PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016/2019
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud', 'opennebula'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 968 cloud-init 19.4 or newer.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 971 dhcp on IPv4.
 972 EODESCR
 973 };
 974 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 975
 976 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 977     $confdesc->{"net$i"} = $netdesc;
 978     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 979 }
 980
 981 foreach my $key (keys %$confdesc_cloudinit) {
 982     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 983 }
 984
 985 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 986 sub verify_volume_id_or_qm_path {
 987     my ($volid, $noerr) = @_;
 988
 989     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 990         return $volid;
 991     }
 992
 993     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 994     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 995     if ($@) {
 996         return if $noerr;
 997         die $@;
 998     }
 999     return $volid;
1000 }
1001
1002 my $usb_fmt = {
1003     host => {
1004         default_key => 1,
1005         type => 'string', format => 'pve-qm-usb-device',
1006         format_description => 'HOSTUSBDEVICE|spice',
1007         description => <<EODESCR,
1008 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1009
1010  'bus-port(.port)*' (decimal numbers) or
1011  'vendor_id:product_id' (hexadeciaml numbers) or
1012  'spice'
1013
1014 You can use the 'lsusb -t' command to list existing usb devices.
1015
1016 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1017 machines - use with special care.
1018
1019 The value 'spice' can be used to add a usb redirection devices for spice.
1020 EODESCR
1021     },
1022     usb3 => {
1023         optional => 1,
1024         type => 'boolean',
1025         description => "Specifies whether if given host option is a USB3 device or port.",
1026         default => 0,
1027     },
1028 };
1029
1030 my $usbdesc = {
1031     optional => 1,
1032     type => 'string', format => $usb_fmt,
1033     description => "Configure an USB device (n is 0 to 4).",
1034 };
1035 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1036
1037 my $serialdesc = {
1038         optional => 1,
1039         type => 'string',
1040         pattern => '(/dev/.+|socket)',
1041         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1042         verbose_description =>  <<EODESCR,
1043 Create a serial device inside the VM (n is 0 to 3), and pass through a
1044 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1045 host side (use 'qm terminal' to open a terminal connection).
1046
1047 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1048 use with special care.
1049
1050 CAUTION: Experimental! User reported problems with this option.
1051 EODESCR
1052 };
1053
1054 my $paralleldesc= {
1055         optional => 1,
1056         type => 'string',
1057         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1058         description =>  "Map host parallel devices (n is 0 to 2).",
1059         verbose_description =>  <<EODESCR,
1060 Map host parallel devices (n is 0 to 2).
1061
1062 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1063 machines - use with special care.
1064
1065 CAUTION: Experimental! User reported problems with this option.
1066 EODESCR
1067 };
1068
1069 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1070     $confdesc->{"parallel$i"} = $paralleldesc;
1071 }
1072
1073 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1074     $confdesc->{"serial$i"} = $serialdesc;
1075 }
1076
1077 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1078     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1079 }
1080
1081 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1082     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1083 }
1084
1085 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1086     $confdesc->{"usb$i"} = $usbdesc;
1087 }
1088
1089 my $boot_fmt = {
1090     legacy => {
1091         optional => 1,
1092         default_key => 1,
1093         type => 'string',
1094         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1095                      . " Deprecated, use 'order=' instead.",
1096         pattern => '[acdn]{1,4}',
1097         format_description => "[acdn]{1,4}",
1098
1099         # note: this is also the fallback if boot: is not given at all
1100         default => 'cdn',
1101     },
1102     order => {
1103         optional => 1,
1104         type => 'string',
1105         format => 'pve-qm-bootdev-list',
1106         format_description => "device[;device...]",
1107         description => <<EODESC,
1108 The guest will attempt to boot from devices in the order they appear here.
1109
1110 Disks, optical drives and passed-through storage USB devices will be directly
1111 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1112 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1113
1114 Note that only devices in this list will be marked as bootable and thus loaded
1115 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1116 (e.g. software-raid), you need to specify all of them here.
1117
1118 Overrides the deprecated 'legacy=[acdn]*' value when given.
1119 EODESC
1120     },
1121 };
1122 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1123
1124 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1125 sub verify_bootdev {
1126     my ($dev, $noerr) = @_;
1127
1128     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1129
1130     my $check = sub {
1131         my ($base) = @_;
1132         return 0 if $dev !~ m/^$base\d+$/;
1133         return 0 if !$confdesc->{$dev};
1134         return 1;
1135     };
1136
1137     return $dev if $check->("net");
1138     return $dev if $check->("usb");
1139     return $dev if $check->("hostpci");
1140
1141     return if $noerr;
1142     die "invalid boot device '$dev'\n";
1143 }
1144
1145 sub print_bootorder {
1146     my ($devs) = @_;
1147     return "" if !@$devs;
1148     my $data = { order => join(';', @$devs) };
1149     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1150 }
1151
1152 my $kvm_api_version = 0;
1153
1154 sub kvm_version {
1155     return $kvm_api_version if $kvm_api_version;
1156
1157     open my $fh, '<', '/dev/kvm' or return;
1158
1159     # 0xae00 => KVM_GET_API_VERSION
1160     $kvm_api_version = ioctl($fh, 0xae00, 0);
1161     close($fh);
1162
1163     return $kvm_api_version;
1164 }
1165
1166 my $kvm_user_version = {};
1167 my $kvm_mtime = {};
1168
1169 sub kvm_user_version {
1170     my ($binary) = @_;
1171
1172     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1173     my $st = stat($binary);
1174
1175     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1176     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1177         $cachedmtime == $st->mtime;
1178
1179     $kvm_user_version->{$binary} = 'unknown';
1180     $kvm_mtime->{$binary} = $st->mtime;
1181
1182     my $code = sub {
1183         my $line = shift;
1184         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1185             $kvm_user_version->{$binary} = $2;
1186         }
1187     };
1188
1189     eval { run_command([$binary, '--version'], outfunc => $code); };
1190     warn $@ if $@;
1191
1192     return $kvm_user_version->{$binary};
1193
1194 }
1195 my sub extract_version {
1196     my ($machine_type, $version) = @_;
1197     $version = kvm_user_version() if !defined($version);
1198     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1199 }
1200
1201 sub kernel_has_vhost_net {
1202     return -c '/dev/vhost-net';
1203 }
1204
1205 sub option_exists {
1206     my $key = shift;
1207     return defined($confdesc->{$key});
1208 }
1209
1210 my $cdrom_path;
1211 sub get_cdrom_path {
1212
1213     return  $cdrom_path if $cdrom_path;
1214
1215     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1216     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1217     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1218 }
1219
1220 sub get_iso_path {
1221     my ($storecfg, $vmid, $cdrom) = @_;
1222
1223     if ($cdrom eq 'cdrom') {
1224         return get_cdrom_path();
1225     } elsif ($cdrom eq 'none') {
1226         return '';
1227     } elsif ($cdrom =~ m|^/|) {
1228         return $cdrom;
1229     } else {
1230         return PVE::Storage::path($storecfg, $cdrom);
1231     }
1232 }
1233
1234 # try to convert old style file names to volume IDs
1235 sub filename_to_volume_id {
1236     my ($vmid, $file, $media) = @_;
1237
1238      if (!($file eq 'none' || $file eq 'cdrom' ||
1239           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1240
1241         return if $file =~ m|/|;
1242
1243         if ($media && $media eq 'cdrom') {
1244             $file = "local:iso/$file";
1245         } else {
1246             $file = "local:$vmid/$file";
1247         }
1248     }
1249
1250     return $file;
1251 }
1252
1253 sub verify_media_type {
1254     my ($opt, $vtype, $media) = @_;
1255
1256     return if !$media;
1257
1258     my $etype;
1259     if ($media eq 'disk') {
1260         $etype = 'images';
1261     } elsif ($media eq 'cdrom') {
1262         $etype = 'iso';
1263     } else {
1264         die "internal error";
1265     }
1266
1267     return if ($vtype eq $etype);
1268
1269     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1270 }
1271
1272 sub cleanup_drive_path {
1273     my ($opt, $storecfg, $drive) = @_;
1274
1275     # try to convert filesystem paths to volume IDs
1276
1277     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1278         ($drive->{file} !~ m|^/dev/.+|) &&
1279         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1280         ($drive->{file} !~ m/^\d+$/)) {
1281         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1282         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1283             if !$vtype;
1284         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1285         verify_media_type($opt, $vtype, $drive->{media});
1286         $drive->{file} = $volid;
1287     }
1288
1289     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1290 }
1291
1292 sub parse_hotplug_features {
1293     my ($data) = @_;
1294
1295     my $res = {};
1296
1297     return $res if $data eq '0';
1298
1299     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1300
1301     foreach my $feature (PVE::Tools::split_list($data)) {
1302         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1303             $res->{$1} = 1;
1304         } else {
1305             die "invalid hotplug feature '$feature'\n";
1306         }
1307     }
1308     return $res;
1309 }
1310
1311 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1312 sub pve_verify_hotplug_features {
1313     my ($value, $noerr) = @_;
1314
1315     return $value if parse_hotplug_features($value);
1316
1317     return if $noerr;
1318
1319     die "unable to parse hotplug option\n";
1320 }
1321
1322 sub scsi_inquiry {
1323     my($fh, $noerr) = @_;
1324
1325     my $SG_IO = 0x2285;
1326     my $SG_GET_VERSION_NUM = 0x2282;
1327
1328     my $versionbuf = "\x00" x 8;
1329     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1330     if (!$ret) {
1331         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1332         return;
1333     }
1334     my $version = unpack("I", $versionbuf);
1335     if ($version < 30000) {
1336         die "scsi generic interface too old\n"  if !$noerr;
1337         return;
1338     }
1339
1340     my $buf = "\x00" x 36;
1341     my $sensebuf = "\x00" x 8;
1342     my $cmd = pack("C x3 C x1", 0x12, 36);
1343
1344     # see /usr/include/scsi/sg.h
1345     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1346
1347     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1348                       length($sensebuf), 0, length($buf), $buf,
1349                       $cmd, $sensebuf, 6000);
1350
1351     $ret = ioctl($fh, $SG_IO, $packet);
1352     if (!$ret) {
1353         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1354         return;
1355     }
1356
1357     my @res = unpack($sg_io_hdr_t, $packet);
1358     if ($res[17] || $res[18]) {
1359         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1360         return;
1361     }
1362
1363     my $res = {};
1364     (my $byte0, my $byte1, $res->{vendor},
1365      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1366
1367     $res->{removable} = $byte1 & 128 ? 1 : 0;
1368     $res->{type} = $byte0 & 31;
1369
1370     return $res;
1371 }
1372
1373 sub path_is_scsi {
1374     my ($path) = @_;
1375
1376     my $fh = IO::File->new("+<$path") || return;
1377     my $res = scsi_inquiry($fh, 1);
1378     close($fh);
1379
1380     return $res;
1381 }
1382
1383 sub print_tabletdevice_full {
1384     my ($conf, $arch) = @_;
1385
1386     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1387
1388     # we use uhci for old VMs because tablet driver was buggy in older qemu
1389     my $usbbus;
1390     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1391         $usbbus = 'ehci';
1392     } else {
1393         $usbbus = 'uhci';
1394     }
1395
1396     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1397 }
1398
1399 sub print_keyboarddevice_full {
1400     my ($conf, $arch, $machine) = @_;
1401
1402     return if $arch ne 'aarch64';
1403
1404     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1405 }
1406
1407 my sub get_drive_id {
1408     my ($drive) = @_;
1409     return "$drive->{interface}$drive->{index}";
1410 }
1411
1412 sub print_drivedevice_full {
1413     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1414
1415     my $device = '';
1416     my $maxdev = 0;
1417
1418     my $drive_id = get_drive_id($drive);
1419     if ($drive->{interface} eq 'virtio') {
1420         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1421         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1422         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1423     } elsif ($drive->{interface} eq 'scsi') {
1424
1425         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1426         my $unit = $drive->{index} % $maxdev;
1427         my $devicetype = 'hd';
1428         my $path = '';
1429         if (drive_is_cdrom($drive)) {
1430             $devicetype = 'cd';
1431         } else {
1432             if ($drive->{file} =~ m|^/|) {
1433                 $path = $drive->{file};
1434                 if (my $info = path_is_scsi($path)) {
1435                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1436                         $devicetype = 'block';
1437                     } elsif ($info->{type} == 1) { # tape
1438                         $devicetype = 'generic';
1439                     }
1440                 }
1441             } else {
1442                  $path = PVE::Storage::path($storecfg, $drive->{file});
1443             }
1444
1445             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1446             my $version = extract_version($machine_type, kvm_user_version());
1447             if ($path =~ m/^iscsi\:\/\// &&
1448                !min_version($version, 4, 1)) {
1449                 $devicetype = 'generic';
1450             }
1451         }
1452
1453         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1454             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1455         } else {
1456             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1457                 .",lun=$drive->{index}";
1458         }
1459         $device .= ",drive=drive-$drive_id,id=$drive_id";
1460
1461         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1462             $device .= ",rotation_rate=1";
1463         }
1464         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1465
1466     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1467         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1468         my $controller = int($drive->{index} / $maxdev);
1469         my $unit = $drive->{index} % $maxdev;
1470         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1471
1472         $device = "ide-$devicetype";
1473         if ($drive->{interface} eq 'ide') {
1474             $device .= ",bus=ide.$controller,unit=$unit";
1475         } else {
1476             $device .= ",bus=ahci$controller.$unit";
1477         }
1478         $device .= ",drive=drive-$drive_id,id=$drive_id";
1479
1480         if ($devicetype eq 'hd') {
1481             if (my $model = $drive->{model}) {
1482                 $model = URI::Escape::uri_unescape($model);
1483                 $device .= ",model=$model";
1484             }
1485             if ($drive->{ssd}) {
1486                 $device .= ",rotation_rate=1";
1487             }
1488         }
1489         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1490     } elsif ($drive->{interface} eq 'usb') {
1491         die "implement me";
1492         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1493     } else {
1494         die "unsupported interface type";
1495     }
1496
1497     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1498
1499     if (my $serial = $drive->{serial}) {
1500         $serial = URI::Escape::uri_unescape($serial);
1501         $device .= ",serial=$serial";
1502     }
1503
1504
1505     return $device;
1506 }
1507
1508 sub get_initiator_name {
1509     my $initiator;
1510
1511     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1512     while (defined(my $line = <$fh>)) {
1513         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1514         $initiator = $1;
1515         last;
1516     }
1517     $fh->close();
1518
1519     return $initiator;
1520 }
1521
1522 sub print_drive_commandline_full {
1523     my ($storecfg, $vmid, $drive, $pbs_name) = @_;
1524
1525     my $path;
1526     my $volid = $drive->{file};
1527     my $format = $drive->{format};
1528     my $drive_id = get_drive_id($drive);
1529
1530     if (drive_is_cdrom($drive)) {
1531         $path = get_iso_path($storecfg, $vmid, $volid);
1532         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1533     } else {
1534         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1535         if ($storeid) {
1536             $path = PVE::Storage::path($storecfg, $volid);
1537             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1538             $format //= qemu_img_format($scfg, $volname);
1539         } else {
1540             $path = $volid;
1541             $format //= "raw";
1542         }
1543    }
1544
1545    my $is_rbd = $path =~ m/^rbd:/;
1546
1547     my $opts = '';
1548     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1549     foreach my $o (@qemu_drive_options) {
1550         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1551     }
1552
1553     # snapshot only accepts on|off
1554     if (defined($drive->{snapshot})) {
1555         my $v = $drive->{snapshot} ? 'on' : 'off';
1556         $opts .= ",snapshot=$v";
1557     }
1558
1559     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1560         my ($dir, $qmpname) = @$type;
1561         if (my $v = $drive->{"mbps$dir"}) {
1562             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1563         }
1564         if (my $v = $drive->{"mbps${dir}_max"}) {
1565             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1566         }
1567         if (my $v = $drive->{"bps${dir}_max_length"}) {
1568             $opts .= ",throttling.bps$qmpname-max-length=$v";
1569         }
1570         if (my $v = $drive->{"iops${dir}"}) {
1571             $opts .= ",throttling.iops$qmpname=$v";
1572         }
1573         if (my $v = $drive->{"iops${dir}_max"}) {
1574             $opts .= ",throttling.iops$qmpname-max=$v";
1575         }
1576         if (my $v = $drive->{"iops${dir}_max_length"}) {
1577             $opts .= ",throttling.iops$qmpname-max-length=$v";
1578         }
1579     }
1580
1581     if ($pbs_name) {
1582         $format = "rbd" if $is_rbd;
1583         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1584             if !$format;
1585         $opts .= ",format=alloc-track,file.driver=$format";
1586     } elsif ($format) {
1587         $opts .= ",format=$format";
1588     }
1589
1590     my $cache_direct = 0;
1591
1592     if (my $cache = $drive->{cache}) {
1593         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1594     } elsif (!drive_is_cdrom($drive)) {
1595         $opts .= ",cache=none";
1596         $cache_direct = 1;
1597     }
1598
1599     # aio native works only with O_DIRECT
1600     if (!$drive->{aio}) {
1601         if($cache_direct) {
1602             $opts .= ",aio=native";
1603         } else {
1604             $opts .= ",aio=threads";
1605         }
1606     }
1607
1608     if (!drive_is_cdrom($drive)) {
1609         my $detectzeroes;
1610         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1611             $detectzeroes = 'off';
1612         } elsif ($drive->{discard}) {
1613             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1614         } else {
1615             # This used to be our default with discard not being specified:
1616             $detectzeroes = 'on';
1617         }
1618
1619         # note: 'detect-zeroes' works per blockdev and we want it to persist
1620         # after the alloc-track is removed, so put it on 'file' directly
1621         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1622         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1623     }
1624
1625     if ($pbs_name) {
1626         $opts .= ",backing=$pbs_name";
1627         $opts .= ",auto-remove=on";
1628     }
1629
1630     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1631     my $file_param = "file";
1632     if ($pbs_name) {
1633         # non-rbd drivers require the underlying file to be a seperate block
1634         # node, so add a second .file indirection
1635         $file_param .= ".file" if !$is_rbd;
1636         $file_param .= ".filename";
1637     }
1638     my $pathinfo = $path ? "$file_param=$path," : '';
1639
1640     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1641 }
1642
1643 sub print_pbs_blockdev {
1644     my ($pbs_conf, $pbs_name) = @_;
1645     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1646     $blockdev .= ",repository=$pbs_conf->{repository}";
1647     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1648     $blockdev .= ",archive=$pbs_conf->{archive}";
1649     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1650     return $blockdev;
1651 }
1652
1653 sub print_netdevice_full {
1654     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1655
1656     my $device = $net->{model};
1657     if ($net->{model} eq 'virtio') {
1658          $device = 'virtio-net-pci';
1659      };
1660
1661     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1662     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1663     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1664         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1665         # and out of each queue plus one config interrupt and control vector queue
1666         my $vectors = $net->{queues} * 2 + 2;
1667         $tmpstr .= ",vectors=$vectors,mq=on";
1668     }
1669     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1670
1671     if (my $mtu = $net->{mtu}) {
1672         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1673             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1674             if ($mtu == 1) {
1675                  $mtu = $bridge_mtu;
1676             } elsif ($mtu < 576) {
1677                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1678             } elsif ($mtu > $bridge_mtu) {
1679                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1680             }
1681             $tmpstr .= ",host_mtu=$mtu";
1682         } else {
1683             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1684         }
1685     }
1686
1687     if ($use_old_bios_files) {
1688         my $romfile;
1689         if ($device eq 'virtio-net-pci') {
1690             $romfile = 'pxe-virtio.rom';
1691         } elsif ($device eq 'e1000') {
1692             $romfile = 'pxe-e1000.rom';
1693         } elsif ($device eq 'ne2k') {
1694             $romfile = 'pxe-ne2k_pci.rom';
1695         } elsif ($device eq 'pcnet') {
1696             $romfile = 'pxe-pcnet.rom';
1697         } elsif ($device eq 'rtl8139') {
1698             $romfile = 'pxe-rtl8139.rom';
1699         }
1700         $tmpstr .= ",romfile=$romfile" if $romfile;
1701     }
1702
1703     return $tmpstr;
1704 }
1705
1706 sub print_netdev_full {
1707     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1708
1709     my $i = '';
1710     if ($netid =~ m/^net(\d+)$/) {
1711         $i = int($1);
1712     }
1713
1714     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1715
1716     my $ifname = "tap${vmid}i$i";
1717
1718     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1719     die "interface name '$ifname' is too long (max 15 character)\n"
1720         if length($ifname) >= 16;
1721
1722     my $vhostparam = '';
1723     if (is_native($arch)) {
1724         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1725     }
1726
1727     my $vmname = $conf->{name} || "vm$vmid";
1728
1729     my $netdev = "";
1730     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1731
1732     if ($net->{bridge}) {
1733         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1734             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1735     } else {
1736         $netdev = "type=user,id=$netid,hostname=$vmname";
1737     }
1738
1739     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1740
1741     return $netdev;
1742 }
1743
1744 my $vga_map = {
1745     'cirrus' => 'cirrus-vga',
1746     'std' => 'VGA',
1747     'vmware' => 'vmware-svga',
1748     'virtio' => 'virtio-vga',
1749 };
1750
1751 sub print_vga_device {
1752     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1753
1754     my $type = $vga_map->{$vga->{type}};
1755     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1756         $type = 'virtio-gpu';
1757     }
1758     my $vgamem_mb = $vga->{memory};
1759
1760     my $max_outputs = '';
1761     if ($qxlnum) {
1762         $type = $id ? 'qxl' : 'qxl-vga';
1763
1764         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1765             # set max outputs so linux can have up to 4 qxl displays with one device
1766             if (min_version($machine_version, 4, 1)) {
1767                 $max_outputs = ",max_outputs=4";
1768             }
1769         }
1770     }
1771
1772     die "no devicetype for $vga->{type}\n" if !$type;
1773
1774     my $memory = "";
1775     if ($vgamem_mb) {
1776         if ($vga->{type} eq 'virtio') {
1777             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1778             $memory = ",max_hostmem=$bytes";
1779         } elsif ($qxlnum) {
1780             # from https://www.spice-space.org/multiple-monitors.html
1781             $memory = ",vgamem_mb=$vga->{memory}";
1782             my $ram = $vgamem_mb * 4;
1783             my $vram = $vgamem_mb * 2;
1784             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1785         } else {
1786             $memory = ",vgamem_mb=$vga->{memory}";
1787         }
1788     } elsif ($qxlnum && $id) {
1789         $memory = ",ram_size=67108864,vram_size=33554432";
1790     }
1791
1792     my $edidoff = "";
1793     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1794         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1795     }
1796
1797     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1798     my $vgaid = "vga" . ($id // '');
1799     my $pciaddr;
1800
1801     if ($q35 && $vgaid eq 'vga') {
1802         # the first display uses pcie.0 bus on q35 machines
1803         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1804     } else {
1805         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1806     }
1807
1808     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1809 }
1810
1811 sub parse_number_sets {
1812     my ($set) = @_;
1813     my $res = [];
1814     foreach my $part (split(/;/, $set)) {
1815         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1816             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1817             push @$res, [ $1, $2 ];
1818         } else {
1819             die "invalid range: $part\n";
1820         }
1821     }
1822     return $res;
1823 }
1824
1825 sub parse_numa {
1826     my ($data) = @_;
1827
1828     my $res = parse_property_string($numa_fmt, $data);
1829     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1830     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1831     return $res;
1832 }
1833
1834 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1835 sub parse_net {
1836     my ($data) = @_;
1837
1838     my $res = eval { parse_property_string($net_fmt, $data) };
1839     if ($@) {
1840         warn $@;
1841         return;
1842     }
1843     if (!defined($res->{macaddr})) {
1844         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1845         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1846     }
1847     return $res;
1848 }
1849
1850 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1851 sub parse_ipconfig {
1852     my ($data) = @_;
1853
1854     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1855     if ($@) {
1856         warn $@;
1857         return;
1858     }
1859
1860     if ($res->{gw} && !$res->{ip}) {
1861         warn 'gateway specified without specifying an IP address';
1862         return;
1863     }
1864     if ($res->{gw6} && !$res->{ip6}) {
1865         warn 'IPv6 gateway specified without specifying an IPv6 address';
1866         return;
1867     }
1868     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1869         warn 'gateway specified together with DHCP';
1870         return;
1871     }
1872     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1873         # gw6 + auto/dhcp
1874         warn "IPv6 gateway specified together with $res->{ip6} address";
1875         return;
1876     }
1877
1878     if (!$res->{ip} && !$res->{ip6}) {
1879         return { ip => 'dhcp', ip6 => 'dhcp' };
1880     }
1881
1882     return $res;
1883 }
1884
1885 sub print_net {
1886     my $net = shift;
1887
1888     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1889 }
1890
1891 sub add_random_macs {
1892     my ($settings) = @_;
1893
1894     foreach my $opt (keys %$settings) {
1895         next if $opt !~ m/^net(\d+)$/;
1896         my $net = parse_net($settings->{$opt});
1897         next if !$net;
1898         $settings->{$opt} = print_net($net);
1899     }
1900 }
1901
1902 sub vm_is_volid_owner {
1903     my ($storecfg, $vmid, $volid) = @_;
1904
1905     if ($volid !~  m|^/|) {
1906         my ($path, $owner);
1907         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1908         if ($owner && ($owner == $vmid)) {
1909             return 1;
1910         }
1911     }
1912
1913     return;
1914 }
1915
1916 sub vmconfig_register_unused_drive {
1917     my ($storecfg, $vmid, $conf, $drive) = @_;
1918
1919     if (drive_is_cloudinit($drive)) {
1920         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1921         warn $@ if $@;
1922     } elsif (!drive_is_cdrom($drive)) {
1923         my $volid = $drive->{file};
1924         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1925             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1926         }
1927     }
1928 }
1929
1930 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1931 my $smbios1_fmt = {
1932     uuid => {
1933         type => 'string',
1934         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1935         format_description => 'UUID',
1936         description => "Set SMBIOS1 UUID.",
1937         optional => 1,
1938     },
1939     version => {
1940         type => 'string',
1941         pattern => '[A-Za-z0-9+\/]+={0,2}',
1942         format_description => 'Base64 encoded string',
1943         description => "Set SMBIOS1 version.",
1944         optional => 1,
1945     },
1946     serial => {
1947         type => 'string',
1948         pattern => '[A-Za-z0-9+\/]+={0,2}',
1949         format_description => 'Base64 encoded string',
1950         description => "Set SMBIOS1 serial number.",
1951         optional => 1,
1952     },
1953     manufacturer => {
1954         type => 'string',
1955         pattern => '[A-Za-z0-9+\/]+={0,2}',
1956         format_description => 'Base64 encoded string',
1957         description => "Set SMBIOS1 manufacturer.",
1958         optional => 1,
1959     },
1960     product => {
1961         type => 'string',
1962         pattern => '[A-Za-z0-9+\/]+={0,2}',
1963         format_description => 'Base64 encoded string',
1964         description => "Set SMBIOS1 product ID.",
1965         optional => 1,
1966     },
1967     sku => {
1968         type => 'string',
1969         pattern => '[A-Za-z0-9+\/]+={0,2}',
1970         format_description => 'Base64 encoded string',
1971         description => "Set SMBIOS1 SKU string.",
1972         optional => 1,
1973     },
1974     family => {
1975         type => 'string',
1976         pattern => '[A-Za-z0-9+\/]+={0,2}',
1977         format_description => 'Base64 encoded string',
1978         description => "Set SMBIOS1 family string.",
1979         optional => 1,
1980     },
1981     base64 => {
1982         type => 'boolean',
1983         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1984         optional => 1,
1985     },
1986 };
1987
1988 sub parse_smbios1 {
1989     my ($data) = @_;
1990
1991     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1992     warn $@ if $@;
1993     return $res;
1994 }
1995
1996 sub print_smbios1 {
1997     my ($smbios1) = @_;
1998     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1999 }
2000
2001 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2002
2003 sub parse_watchdog {
2004     my ($value) = @_;
2005
2006     return if !$value;
2007
2008     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2009     warn $@ if $@;
2010     return $res;
2011 }
2012
2013 sub parse_guest_agent {
2014     my ($conf) = @_;
2015
2016     return {} if !defined($conf->{agent});
2017
2018     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2019     warn $@ if $@;
2020
2021     # if the agent is disabled ignore the other potentially set properties
2022     return {} if !$res->{enabled};
2023     return $res;
2024 }
2025
2026 sub get_qga_key {
2027     my ($conf, $key) = @_;
2028     return undef if !defined($conf->{agent});
2029
2030     my $agent = parse_guest_agent($conf);
2031     return $agent->{$key};
2032 }
2033
2034 sub parse_vga {
2035     my ($value) = @_;
2036
2037     return {} if !$value;
2038     my $res = eval { parse_property_string($vga_fmt, $value) };
2039     warn $@ if $@;
2040     return $res;
2041 }
2042
2043 sub parse_rng {
2044     my ($value) = @_;
2045
2046     return if !$value;
2047
2048     my $res = eval { parse_property_string($rng_fmt, $value) };
2049     warn $@ if $@;
2050     return $res;
2051 }
2052
2053 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2054 sub verify_usb_device {
2055     my ($value, $noerr) = @_;
2056
2057     return $value if parse_usb_device($value);
2058
2059     return if $noerr;
2060
2061     die "unable to parse usb device\n";
2062 }
2063
2064 # add JSON properties for create and set function
2065 sub json_config_properties {
2066     my $prop = shift;
2067
2068     foreach my $opt (keys %$confdesc) {
2069         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2070             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2071         $prop->{$opt} = $confdesc->{$opt};
2072     }
2073
2074     return $prop;
2075 }
2076
2077 # return copy of $confdesc_cloudinit to generate documentation
2078 sub cloudinit_config_properties {
2079
2080     return dclone($confdesc_cloudinit);
2081 }
2082
2083 sub check_type {
2084     my ($key, $value) = @_;
2085
2086     die "unknown setting '$key'\n" if !$confdesc->{$key};
2087
2088     my $type = $confdesc->{$key}->{type};
2089
2090     if (!defined($value)) {
2091         die "got undefined value\n";
2092     }
2093
2094     if ($value =~ m/[\n\r]/) {
2095         die "property contains a line feed\n";
2096     }
2097
2098     if ($type eq 'boolean') {
2099         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2100         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2101         die "type check ('boolean') failed - got '$value'\n";
2102     } elsif ($type eq 'integer') {
2103         return int($1) if $value =~ m/^(\d+)$/;
2104         die "type check ('integer') failed - got '$value'\n";
2105     } elsif ($type eq 'number') {
2106         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2107         die "type check ('number') failed - got '$value'\n";
2108     } elsif ($type eq 'string') {
2109         if (my $fmt = $confdesc->{$key}->{format}) {
2110             PVE::JSONSchema::check_format($fmt, $value);
2111             return $value;
2112         }
2113         $value =~ s/^\"(.*)\"$/$1/;
2114         return $value;
2115     } else {
2116         die "internal error"
2117     }
2118 }
2119
2120 sub destroy_vm {
2121     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2122
2123     my $conf = PVE::QemuConfig->load_config($vmid);
2124
2125     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2126
2127     if ($conf->{template}) {
2128         # check if any base image is still used by a linked clone
2129         PVE::QemuConfig->foreach_volume($conf, sub {
2130                 my ($ds, $drive) = @_;
2131                 return if drive_is_cdrom($drive);
2132
2133                 my $volid = $drive->{file};
2134                 return if !$volid || $volid =~ m|^/|;
2135
2136                 die "base volume '$volid' is still in use by linked cloned\n"
2137                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2138
2139         });
2140     }
2141
2142     # only remove disks owned by this VM (referenced in the config)
2143     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2144         my ($ds, $drive) = @_;
2145         return if drive_is_cdrom($drive, 1);
2146
2147         my $volid = $drive->{file};
2148         return if !$volid || $volid =~ m|^/|;
2149
2150         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2151         return if !$path || !$owner || ($owner != $vmid);
2152
2153         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2154         warn "Could not remove disk '$volid', check manually: $@" if $@;
2155     });
2156
2157     if ($purge_unreferenced) { # also remove unreferenced disk
2158         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2159         PVE::Storage::foreach_volid($vmdisks, sub {
2160             my ($volid, $sid, $volname, $d) = @_;
2161             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2162             warn $@ if $@;
2163         });
2164     }
2165
2166     if (defined $replacement_conf) {
2167         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2168     } else {
2169         PVE::QemuConfig->destroy_config($vmid);
2170     }
2171 }
2172
2173 sub parse_vm_config {
2174     my ($filename, $raw) = @_;
2175
2176     return if !defined($raw);
2177
2178     my $res = {
2179         digest => Digest::SHA::sha1_hex($raw),
2180         snapshots => {},
2181         pending => {},
2182     };
2183
2184     $filename =~ m|/qemu-server/(\d+)\.conf$|
2185         || die "got strange filename '$filename'";
2186
2187     my $vmid = $1;
2188
2189     my $conf = $res;
2190     my $descr;
2191     my $section = '';
2192
2193     my @lines = split(/\n/, $raw);
2194     foreach my $line (@lines) {
2195         next if $line =~ m/^\s*$/;
2196
2197         if ($line =~ m/^\[PENDING\]\s*$/i) {
2198             $section = 'pending';
2199             if (defined($descr)) {
2200                 $descr =~ s/\s+$//;
2201                 $conf->{description} = $descr;
2202             }
2203             $descr = undef;
2204             $conf = $res->{$section} = {};
2205             next;
2206
2207         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2208             $section = $1;
2209             if (defined($descr)) {
2210                 $descr =~ s/\s+$//;
2211                 $conf->{description} = $descr;
2212             }
2213             $descr = undef;
2214             $conf = $res->{snapshots}->{$section} = {};
2215             next;
2216         }
2217
2218         if ($line =~ m/^\#(.*)\s*$/) {
2219             $descr = '' if !defined($descr);
2220             $descr .= PVE::Tools::decode_text($1) . "\n";
2221             next;
2222         }
2223
2224         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2225             $descr = '' if !defined($descr);
2226             $descr .= PVE::Tools::decode_text($2);
2227         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2228             $conf->{snapstate} = $1;
2229         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2230             my $key = $1;
2231             my $value = $2;
2232             $conf->{$key} = $value;
2233         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2234             my $value = $1;
2235             if ($section eq 'pending') {
2236                 $conf->{delete} = $value; # we parse this later
2237             } else {
2238                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2239             }
2240         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2241             my $key = $1;
2242             my $value = $2;
2243             eval { $value = check_type($key, $value); };
2244             if ($@) {
2245                 warn "vm $vmid - unable to parse value of '$key' - $@";
2246             } else {
2247                 $key = 'ide2' if $key eq 'cdrom';
2248                 my $fmt = $confdesc->{$key}->{format};
2249                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2250                     my $v = parse_drive($key, $value);
2251                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2252                         $v->{file} = $volid;
2253                         $value = print_drive($v);
2254                     } else {
2255                         warn "vm $vmid - unable to parse value of '$key'\n";
2256                         next;
2257                     }
2258                 }
2259
2260                 $conf->{$key} = $value;
2261             }
2262         } else {
2263             warn "vm $vmid - unable to parse config: $line\n";
2264         }
2265     }
2266
2267     if (defined($descr)) {
2268         $descr =~ s/\s+$//;
2269         $conf->{description} = $descr;
2270     }
2271     delete $res->{snapstate}; # just to be sure
2272
2273     return $res;
2274 }
2275
2276 sub write_vm_config {
2277     my ($filename, $conf) = @_;
2278
2279     delete $conf->{snapstate}; # just to be sure
2280
2281     if ($conf->{cdrom}) {
2282         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2283         $conf->{ide2} = $conf->{cdrom};
2284         delete $conf->{cdrom};
2285     }
2286
2287     # we do not use 'smp' any longer
2288     if ($conf->{sockets}) {
2289         delete $conf->{smp};
2290     } elsif ($conf->{smp}) {
2291         $conf->{sockets} = $conf->{smp};
2292         delete $conf->{cores};
2293         delete $conf->{smp};
2294     }
2295
2296     my $used_volids = {};
2297
2298     my $cleanup_config = sub {
2299         my ($cref, $pending, $snapname) = @_;
2300
2301         foreach my $key (keys %$cref) {
2302             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2303                 $key eq 'snapstate' || $key eq 'pending';
2304             my $value = $cref->{$key};
2305             if ($key eq 'delete') {
2306                 die "propertry 'delete' is only allowed in [PENDING]\n"
2307                     if !$pending;
2308                 # fixme: check syntax?
2309                 next;
2310             }
2311             eval { $value = check_type($key, $value); };
2312             die "unable to parse value of '$key' - $@" if $@;
2313
2314             $cref->{$key} = $value;
2315
2316             if (!$snapname && is_valid_drivename($key)) {
2317                 my $drive = parse_drive($key, $value);
2318                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2319             }
2320         }
2321     };
2322
2323     &$cleanup_config($conf);
2324
2325     &$cleanup_config($conf->{pending}, 1);
2326
2327     foreach my $snapname (keys %{$conf->{snapshots}}) {
2328         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2329         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2330     }
2331
2332     # remove 'unusedX' settings if we re-add a volume
2333     foreach my $key (keys %$conf) {
2334         my $value = $conf->{$key};
2335         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2336             delete $conf->{$key};
2337         }
2338     }
2339
2340     my $generate_raw_config = sub {
2341         my ($conf, $pending) = @_;
2342
2343         my $raw = '';
2344
2345         # add description as comment to top of file
2346         if (defined(my $descr = $conf->{description})) {
2347             if ($descr) {
2348                 foreach my $cl (split(/\n/, $descr)) {
2349                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2350                 }
2351             } else {
2352                 $raw .= "#\n" if $pending;
2353             }
2354         }
2355
2356         foreach my $key (sort keys %$conf) {
2357             next if $key =~ /^(digest|description|pending|snapshots)$/;
2358             $raw .= "$key: $conf->{$key}\n";
2359         }
2360         return $raw;
2361     };
2362
2363     my $raw = &$generate_raw_config($conf);
2364
2365     if (scalar(keys %{$conf->{pending}})){
2366         $raw .= "\n[PENDING]\n";
2367         $raw .= &$generate_raw_config($conf->{pending}, 1);
2368     }
2369
2370     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2371         $raw .= "\n[$snapname]\n";
2372         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2373     }
2374
2375     return $raw;
2376 }
2377
2378 sub load_defaults {
2379
2380     my $res = {};
2381
2382     # we use static defaults from our JSON schema configuration
2383     foreach my $key (keys %$confdesc) {
2384         if (defined(my $default = $confdesc->{$key}->{default})) {
2385             $res->{$key} = $default;
2386         }
2387     }
2388
2389     return $res;
2390 }
2391
2392 sub config_list {
2393     my $vmlist = PVE::Cluster::get_vmlist();
2394     my $res = {};
2395     return $res if !$vmlist || !$vmlist->{ids};
2396     my $ids = $vmlist->{ids};
2397     my $nodename = nodename();
2398
2399     foreach my $vmid (keys %$ids) {
2400         my $d = $ids->{$vmid};
2401         next if !$d->{node} || $d->{node} ne $nodename;
2402         next if !$d->{type} || $d->{type} ne 'qemu';
2403         $res->{$vmid}->{exists} = 1;
2404     }
2405     return $res;
2406 }
2407
2408 # test if VM uses local resources (to prevent migration)
2409 sub check_local_resources {
2410     my ($conf, $noerr) = @_;
2411
2412     my @loc_res = ();
2413
2414     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2415     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2416
2417     push @loc_res, "ivshmem" if $conf->{ivshmem};
2418
2419     foreach my $k (keys %$conf) {
2420         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2421         # sockets are safe: they will recreated be on the target side post-migrate
2422         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2423         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2424     }
2425
2426     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2427
2428     return \@loc_res;
2429 }
2430
2431 # check if used storages are available on all nodes (use by migrate)
2432 sub check_storage_availability {
2433     my ($storecfg, $conf, $node) = @_;
2434
2435     PVE::QemuConfig->foreach_volume($conf, sub {
2436         my ($ds, $drive) = @_;
2437
2438         my $volid = $drive->{file};
2439         return if !$volid;
2440
2441         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2442         return if !$sid;
2443
2444         # check if storage is available on both nodes
2445         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2446         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2447    });
2448 }
2449
2450 # list nodes where all VM images are available (used by has_feature API)
2451 sub shared_nodes {
2452     my ($conf, $storecfg) = @_;
2453
2454     my $nodelist = PVE::Cluster::get_nodelist();
2455     my $nodehash = { map { $_ => 1 } @$nodelist };
2456     my $nodename = nodename();
2457
2458     PVE::QemuConfig->foreach_volume($conf, sub {
2459         my ($ds, $drive) = @_;
2460
2461         my $volid = $drive->{file};
2462         return if !$volid;
2463
2464         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2465         if ($storeid) {
2466             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2467             if ($scfg->{disable}) {
2468                 $nodehash = {};
2469             } elsif (my $avail = $scfg->{nodes}) {
2470                 foreach my $node (keys %$nodehash) {
2471                     delete $nodehash->{$node} if !$avail->{$node};
2472                 }
2473             } elsif (!$scfg->{shared}) {
2474                 foreach my $node (keys %$nodehash) {
2475                     delete $nodehash->{$node} if $node ne $nodename
2476                 }
2477             }
2478         }
2479     });
2480
2481     return $nodehash
2482 }
2483
2484 sub check_local_storage_availability {
2485     my ($conf, $storecfg) = @_;
2486
2487     my $nodelist = PVE::Cluster::get_nodelist();
2488     my $nodehash = { map { $_ => {} } @$nodelist };
2489
2490     PVE::QemuConfig->foreach_volume($conf, sub {
2491         my ($ds, $drive) = @_;
2492
2493         my $volid = $drive->{file};
2494         return if !$volid;
2495
2496         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2497         if ($storeid) {
2498             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2499
2500             if ($scfg->{disable}) {
2501                 foreach my $node (keys %$nodehash) {
2502                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2503                 }
2504             } elsif (my $avail = $scfg->{nodes}) {
2505                 foreach my $node (keys %$nodehash) {
2506                     if (!$avail->{$node}) {
2507                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2508                     }
2509                 }
2510             }
2511         }
2512     });
2513
2514     foreach my $node (values %$nodehash) {
2515         if (my $unavail = $node->{unavailable_storages}) {
2516             $node->{unavailable_storages} = [ sort keys %$unavail ];
2517         }
2518     }
2519
2520     return $nodehash
2521 }
2522
2523 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2524 sub check_running {
2525     my ($vmid, $nocheck, $node) = @_;
2526
2527     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2528     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2529 }
2530
2531 sub vzlist {
2532
2533     my $vzlist = config_list();
2534
2535     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2536
2537     while (defined(my $de = $fd->read)) {
2538         next if $de !~ m/^(\d+)\.pid$/;
2539         my $vmid = $1;
2540         next if !defined($vzlist->{$vmid});
2541         if (my $pid = check_running($vmid)) {
2542             $vzlist->{$vmid}->{pid} = $pid;
2543         }
2544     }
2545
2546     return $vzlist;
2547 }
2548
2549 our $vmstatus_return_properties = {
2550     vmid => get_standard_option('pve-vmid'),
2551     status => {
2552         description => "Qemu process status.",
2553         type => 'string',
2554         enum => ['stopped', 'running'],
2555     },
2556     maxmem => {
2557         description => "Maximum memory in bytes.",
2558         type => 'integer',
2559         optional => 1,
2560         renderer => 'bytes',
2561     },
2562     maxdisk => {
2563         description => "Root disk size in bytes.",
2564         type => 'integer',
2565         optional => 1,
2566         renderer => 'bytes',
2567     },
2568     name => {
2569         description => "VM name.",
2570         type => 'string',
2571         optional => 1,
2572     },
2573     qmpstatus => {
2574         description => "Qemu QMP agent status.",
2575         type => 'string',
2576         optional => 1,
2577     },
2578     pid => {
2579         description => "PID of running qemu process.",
2580         type => 'integer',
2581         optional => 1,
2582     },
2583     uptime => {
2584         description => "Uptime.",
2585         type => 'integer',
2586         optional => 1,
2587         renderer => 'duration',
2588     },
2589     cpus => {
2590         description => "Maximum usable CPUs.",
2591         type => 'number',
2592         optional => 1,
2593     },
2594     lock => {
2595         description => "The current config lock, if any.",
2596         type => 'string',
2597         optional => 1,
2598     },
2599     tags => {
2600         description  => "The current configured tags, if any",
2601         type => 'string',
2602         optional => 1,
2603     },
2604     'running-machine' => {
2605         description => "The currently running machine type (if running).",
2606         type => 'string',
2607         optional => 1,
2608     },
2609     'running-qemu' => {
2610         description => "The currently running QEMU version (if running).",
2611         type => 'string',
2612         optional => 1,
2613     },
2614 };
2615
2616 my $last_proc_pid_stat;
2617
2618 # get VM status information
2619 # This must be fast and should not block ($full == false)
2620 # We only query KVM using QMP if $full == true (this can be slow)
2621 sub vmstatus {
2622     my ($opt_vmid, $full) = @_;
2623
2624     my $res = {};
2625
2626     my $storecfg = PVE::Storage::config();
2627
2628     my $list = vzlist();
2629     my $defaults = load_defaults();
2630
2631     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2632
2633     my $cpucount = $cpuinfo->{cpus} || 1;
2634
2635     foreach my $vmid (keys %$list) {
2636         next if $opt_vmid && ($vmid ne $opt_vmid);
2637
2638         my $conf = PVE::QemuConfig->load_config($vmid);
2639
2640         my $d = { vmid => $vmid };
2641         $d->{pid} = $list->{$vmid}->{pid};
2642
2643         # fixme: better status?
2644         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2645
2646         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2647         if (defined($size)) {
2648             $d->{disk} = 0; # no info available
2649             $d->{maxdisk} = $size;
2650         } else {
2651             $d->{disk} = 0;
2652             $d->{maxdisk} = 0;
2653         }
2654
2655         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2656             * ($conf->{cores} || $defaults->{cores});
2657         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2658         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2659
2660         $d->{name} = $conf->{name} || "VM $vmid";
2661         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2662             : $defaults->{memory}*(1024*1024);
2663
2664         if ($conf->{balloon}) {
2665             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2666             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2667                 : $defaults->{shares};
2668         }
2669
2670         $d->{uptime} = 0;
2671         $d->{cpu} = 0;
2672         $d->{mem} = 0;
2673
2674         $d->{netout} = 0;
2675         $d->{netin} = 0;
2676
2677         $d->{diskread} = 0;
2678         $d->{diskwrite} = 0;
2679
2680         $d->{template} = PVE::QemuConfig->is_template($conf);
2681
2682         $d->{serial} = 1 if conf_has_serial($conf);
2683         $d->{lock} = $conf->{lock} if $conf->{lock};
2684         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2685
2686         $res->{$vmid} = $d;
2687     }
2688
2689     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2690     foreach my $dev (keys %$netdev) {
2691         next if $dev !~ m/^tap([1-9]\d*)i/;
2692         my $vmid = $1;
2693         my $d = $res->{$vmid};
2694         next if !$d;
2695
2696         $d->{netout} += $netdev->{$dev}->{receive};
2697         $d->{netin} += $netdev->{$dev}->{transmit};
2698
2699         if ($full) {
2700             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2701             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2702         }
2703
2704     }
2705
2706     my $ctime = gettimeofday;
2707
2708     foreach my $vmid (keys %$list) {
2709
2710         my $d = $res->{$vmid};
2711         my $pid = $d->{pid};
2712         next if !$pid;
2713
2714         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2715         next if !$pstat; # not running
2716
2717         my $used = $pstat->{utime} + $pstat->{stime};
2718
2719         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2720
2721         if ($pstat->{vsize}) {
2722             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2723         }
2724
2725         my $old = $last_proc_pid_stat->{$pid};
2726         if (!$old) {
2727             $last_proc_pid_stat->{$pid} = {
2728                 time => $ctime,
2729                 used => $used,
2730                 cpu => 0,
2731             };
2732             next;
2733         }
2734
2735         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2736
2737         if ($dtime > 1000) {
2738             my $dutime = $used -  $old->{used};
2739
2740             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2741             $last_proc_pid_stat->{$pid} = {
2742                 time => $ctime,
2743                 used => $used,
2744                 cpu => $d->{cpu},
2745             };
2746         } else {
2747             $d->{cpu} = $old->{cpu};
2748         }
2749     }
2750
2751     return $res if !$full;
2752
2753     my $qmpclient = PVE::QMPClient->new();
2754
2755     my $ballooncb = sub {
2756         my ($vmid, $resp) = @_;
2757
2758         my $info = $resp->{'return'};
2759         return if !$info->{max_mem};
2760
2761         my $d = $res->{$vmid};
2762
2763         # use memory assigned to VM
2764         $d->{maxmem} = $info->{max_mem};
2765         $d->{balloon} = $info->{actual};
2766
2767         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2768             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2769             $d->{freemem} = $info->{free_mem};
2770         }
2771
2772         $d->{ballooninfo} = $info;
2773     };
2774
2775     my $blockstatscb = sub {
2776         my ($vmid, $resp) = @_;
2777         my $data = $resp->{'return'} || [];
2778         my $totalrdbytes = 0;
2779         my $totalwrbytes = 0;
2780
2781         for my $blockstat (@$data) {
2782             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2783             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2784
2785             $blockstat->{device} =~ s/drive-//;
2786             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2787         }
2788         $res->{$vmid}->{diskread} = $totalrdbytes;
2789         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2790     };
2791
2792     my $machinecb = sub {
2793         my ($vmid, $resp) = @_;
2794         my $data = $resp->{'return'} || [];
2795
2796         $res->{$vmid}->{'running-machine'} =
2797             PVE::QemuServer::Machine::current_from_query_machines($data);
2798     };
2799
2800     my $versioncb = sub {
2801         my ($vmid, $resp) = @_;
2802         my $data = $resp->{'return'} // {};
2803         my $version = 'unknown';
2804
2805         if (my $v = $data->{qemu}) {
2806             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2807         }
2808
2809         $res->{$vmid}->{'running-qemu'} = $version;
2810     };
2811
2812     my $statuscb = sub {
2813         my ($vmid, $resp) = @_;
2814
2815         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2816         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2817         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2818         # this fails if ballon driver is not loaded, so this must be
2819         # the last commnand (following command are aborted if this fails).
2820         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2821
2822         my $status = 'unknown';
2823         if (!defined($status = $resp->{'return'}->{status})) {
2824             warn "unable to get VM status\n";
2825             return;
2826         }
2827
2828         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2829     };
2830
2831     foreach my $vmid (keys %$list) {
2832         next if $opt_vmid && ($vmid ne $opt_vmid);
2833         next if !$res->{$vmid}->{pid}; # not running
2834         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2835     }
2836
2837     $qmpclient->queue_execute(undef, 2);
2838
2839     foreach my $vmid (keys %$list) {
2840         next if $opt_vmid && ($vmid ne $opt_vmid);
2841         next if !$res->{$vmid}->{pid}; #not running
2842
2843         # we can't use the $qmpclient since it might have already aborted on
2844         # 'query-balloon', but this might also fail for older versions...
2845         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2846         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2847     }
2848
2849     foreach my $vmid (keys %$list) {
2850         next if $opt_vmid && ($vmid ne $opt_vmid);
2851         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2852     }
2853
2854     return $res;
2855 }
2856
2857 sub conf_has_serial {
2858     my ($conf) = @_;
2859
2860     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2861         if ($conf->{"serial$i"}) {
2862             return 1;
2863         }
2864     }
2865
2866     return 0;
2867 }
2868
2869 sub conf_has_audio {
2870     my ($conf, $id) = @_;
2871
2872     $id //= 0;
2873     my $audio = $conf->{"audio$id"};
2874     return if !defined($audio);
2875
2876     my $audioproperties = parse_property_string($audio_fmt, $audio);
2877     my $audiodriver = $audioproperties->{driver} // 'spice';
2878
2879     return {
2880         dev => $audioproperties->{device},
2881         dev_id => "audiodev$id",
2882         backend => $audiodriver,
2883         backend_id => "$audiodriver-backend${id}",
2884     };
2885 }
2886
2887 sub audio_devs {
2888     my ($audio, $audiopciaddr, $machine_version) = @_;
2889
2890     my $devs = [];
2891
2892     my $id = $audio->{dev_id};
2893     my $audiodev = "";
2894     if (min_version($machine_version, 4, 2)) {
2895         $audiodev = ",audiodev=$audio->{backend_id}";
2896     }
2897
2898     if ($audio->{dev} eq 'AC97') {
2899         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2900     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2901         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2902         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2903         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2904     } else {
2905         die "unkown audio device '$audio->{dev}', implement me!";
2906     }
2907
2908     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2909
2910     return $devs;
2911 }
2912
2913 sub vga_conf_has_spice {
2914     my ($vga) = @_;
2915
2916     my $vgaconf = parse_vga($vga);
2917     my $vgatype = $vgaconf->{type};
2918     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2919
2920     return $1 || 1;
2921 }
2922
2923 sub is_native($) {
2924     my ($arch) = @_;
2925     return get_host_arch() eq $arch;
2926 }
2927
2928 sub get_vm_arch {
2929     my ($conf) = @_;
2930     return $conf->{arch} // get_host_arch();
2931 }
2932
2933 my $default_machines = {
2934     x86_64 => 'pc',
2935     aarch64 => 'virt',
2936 };
2937
2938 sub get_installed_machine_version {
2939     my ($kvmversion) = @_;
2940     $kvmversion = kvm_user_version() if !defined($kvmversion);
2941     $kvmversion =~ m/^(\d+\.\d+)/;
2942     return $1;
2943 }
2944
2945 sub windows_get_pinned_machine_version {
2946     my ($machine, $base_version, $kvmversion) = @_;
2947
2948     my $pin_version = $base_version;
2949     if (!defined($base_version) ||
2950         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2951     ) {
2952         $pin_version = get_installed_machine_version($kvmversion);
2953     }
2954     if (!$machine || $machine eq 'pc') {
2955         $machine = "pc-i440fx-$pin_version";
2956     } elsif ($machine eq 'q35') {
2957         $machine = "pc-q35-$pin_version";
2958     } elsif ($machine eq 'virt') {
2959         $machine = "virt-$pin_version";
2960     } else {
2961         warn "unknown machine type '$machine', not touching that!\n";
2962     }
2963
2964     return $machine;
2965 }
2966
2967 sub get_vm_machine {
2968     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2969
2970     my $machine = $forcemachine || $conf->{machine};
2971
2972     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2973         $kvmversion //= kvm_user_version();
2974         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2975         # layout which confuses windows quite a bit and may result in various regressions..
2976         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2977         if (windows_version($conf->{ostype})) {
2978             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2979         }
2980         $arch //= 'x86_64';
2981         $machine ||= $default_machines->{$arch};
2982         if ($add_pve_version) {
2983             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2984             $machine .= "+pve$pvever";
2985         }
2986     }
2987
2988     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2989         # for version-pinned machines that do not include a pve-version (e.g.
2990         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2991         $machine .= '+pve0';
2992     }
2993
2994     return $machine;
2995 }
2996
2997 sub get_ovmf_files($) {
2998     my ($arch) = @_;
2999
3000     my $ovmf = $OVMF->{$arch}
3001         or die "no OVMF images known for architecture '$arch'\n";
3002
3003     return @$ovmf;
3004 }
3005
3006 my $Arch2Qemu = {
3007     aarch64 => '/usr/bin/qemu-system-aarch64',
3008     x86_64 => '/usr/bin/qemu-system-x86_64',
3009 };
3010 sub get_command_for_arch($) {
3011     my ($arch) = @_;
3012     return '/usr/bin/kvm' if is_native($arch);
3013
3014     my $cmd = $Arch2Qemu->{$arch}
3015         or die "don't know how to emulate architecture '$arch'\n";
3016     return $cmd;
3017 }
3018
3019 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3020 # to use in a QEMU command line (-cpu element), first array_intersect the result
3021 # of query_supported_ with query_understood_. This is necessary because:
3022 #
3023 # a) query_understood_ returns flags the host cannot use and
3024 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3025 #    flags, but CPU settings - with most of them being flags. Those settings
3026 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3027 #
3028 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3029 # expensive. If you need the value returned from this, you can get it much
3030 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3031 # $accel being 'kvm' or 'tcg'.
3032 #
3033 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3034 # changes, automatically populating pmxcfs.
3035 #
3036 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3037 # since kvm and tcg machines support different flags
3038 #
3039 sub query_supported_cpu_flags {
3040     my ($arch) = @_;
3041
3042     $arch //= get_host_arch();
3043     my $default_machine = $default_machines->{$arch};
3044
3045     my $flags = {};
3046
3047     # FIXME: Once this is merged, the code below should work for ARM as well:
3048     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3049     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3050         $arch eq "aarch64";
3051
3052     my $kvm_supported = defined(kvm_version());
3053     my $qemu_cmd = get_command_for_arch($arch);
3054     my $fakevmid = -1;
3055     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3056
3057     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3058     my $query_supported_run_qemu = sub {
3059         my ($kvm) = @_;
3060
3061         my $flags = {};
3062         my $cmd = [
3063             $qemu_cmd,
3064             '-machine', $default_machine,
3065             '-display', 'none',
3066             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3067             '-mon', 'chardev=qmp,mode=control',
3068             '-pidfile', $pidfile,
3069             '-S', '-daemonize'
3070         ];
3071
3072         if (!$kvm) {
3073             push @$cmd, '-accel', 'tcg';
3074         }
3075
3076         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3077         die "QEMU flag querying VM exited with code " . $rc if $rc;
3078
3079         eval {
3080             my $cmd_result = mon_cmd(
3081                 $fakevmid,
3082                 'query-cpu-model-expansion',
3083                 type => 'full',
3084                 model => { name => 'host' }
3085             );
3086
3087             my $props = $cmd_result->{model}->{props};
3088             foreach my $prop (keys %$props) {
3089                 next if $props->{$prop} ne '1';
3090                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3091                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3092                 # We only keep those with underscores, to match /proc/cpuinfo
3093                 $prop =~ s/\.|-/_/g;
3094                 $flags->{$prop} = 1;
3095             }
3096         };
3097         my $err = $@;
3098
3099         # force stop with 10 sec timeout and 'nocheck'
3100         # always stop, even if QMP failed
3101         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3102
3103         die $err if $err;
3104
3105         return [ sort keys %$flags ];
3106     };
3107
3108     # We need to query QEMU twice, since KVM and TCG have different supported flags
3109     PVE::QemuConfig->lock_config($fakevmid, sub {
3110         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3111         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3112
3113         if ($kvm_supported) {
3114             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3115             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3116         }
3117     });
3118
3119     return $flags;
3120 }
3121
3122 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3123 my $understood_cpu_flag_dir = "/usr/share/kvm";
3124 sub query_understood_cpu_flags {
3125     my $arch = get_host_arch();
3126     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3127
3128     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3129         if ! -e $filepath;
3130
3131     my $raw = file_get_contents($filepath);
3132     $raw =~ s/^\s+|\s+$//g;
3133     my @flags = split(/\s+/, $raw);
3134
3135     return \@flags;
3136 }
3137
3138 sub config_to_command {
3139     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3140         $pbs_backing) = @_;
3141
3142     my $cmd = [];
3143     my $globalFlags = [];
3144     my $machineFlags = [];
3145     my $rtcFlags = [];
3146     my $devices = [];
3147     my $pciaddr = '';
3148     my $bridges = {};
3149     my $ostype = $conf->{ostype};
3150     my $winversion = windows_version($ostype);
3151     my $kvm = $conf->{kvm};
3152     my $nodename = nodename();
3153
3154     my $arch = get_vm_arch($conf);
3155     my $kvm_binary = get_command_for_arch($arch);
3156     my $kvmver = kvm_user_version($kvm_binary);
3157
3158     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3159         $kvmver //= "undefined";
3160         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3161     }
3162
3163     my $add_pve_version = min_version($kvmver, 4, 1);
3164
3165     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3166     my $machine_version = extract_version($machine_type, $kvmver);
3167     $kvm //= 1 if is_native($arch);
3168
3169     $machine_version =~ m/(\d+)\.(\d+)/;
3170     my ($machine_major, $machine_minor) = ($1, $2);
3171
3172     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3173         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3174     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3175         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3176             ." please upgrade node '$nodename'\n"
3177     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3178         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3179         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3180             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3181             ." node '$nodename'\n";
3182     }
3183
3184     # if a specific +pve version is required for a feature, use $version_guard
3185     # instead of min_version to allow machines to be run with the minimum
3186     # required version
3187     my $required_pve_version = 0;
3188     my $version_guard = sub {
3189         my ($major, $minor, $pve) = @_;
3190         return 0 if !min_version($machine_version, $major, $minor, $pve);
3191         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3192         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3193         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3194         return 1;
3195     };
3196
3197     if ($kvm && !defined kvm_version()) {
3198         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3199             ." or enable in BIOS.\n";
3200     }
3201
3202     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3203     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3204     my $use_old_bios_files = undef;
3205     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3206
3207     my $cpuunits = defined($conf->{cpuunits}) ?
3208             $conf->{cpuunits} : $defaults->{cpuunits};
3209
3210     push @$cmd, $kvm_binary;
3211
3212     push @$cmd, '-id', $vmid;
3213
3214     my $vmname = $conf->{name} || "vm$vmid";
3215
3216     push @$cmd, '-name', $vmname;
3217
3218     push @$cmd, '-no-shutdown';
3219
3220     my $use_virtio = 0;
3221
3222     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3223     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3224     push @$cmd, '-mon', "chardev=qmp,mode=control";
3225
3226     if (min_version($machine_version, 2, 12)) {
3227         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3228         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3229     }
3230
3231     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3232
3233     push @$cmd, '-daemonize';
3234
3235     if ($conf->{smbios1}) {
3236         my $smbios_conf = parse_smbios1($conf->{smbios1});
3237         if ($smbios_conf->{base64}) {
3238             # Do not pass base64 flag to qemu
3239             delete $smbios_conf->{base64};
3240             my $smbios_string = "";
3241             foreach my $key (keys %$smbios_conf) {
3242                 my $value;
3243                 if ($key eq "uuid") {
3244                     $value = $smbios_conf->{uuid}
3245                 } else {
3246                     $value = decode_base64($smbios_conf->{$key});
3247                 }
3248                 # qemu accepts any binary data, only commas need escaping by double comma
3249                 $value =~ s/,/,,/g;
3250                 $smbios_string .= "," . $key . "=" . $value if $value;
3251             }
3252             push @$cmd, '-smbios', "type=1" . $smbios_string;
3253         } else {
3254             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3255         }
3256     }
3257
3258     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3259         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3260         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3261
3262         my ($path, $format);
3263         if (my $efidisk = $conf->{efidisk0}) {
3264             my $d = parse_drive('efidisk0', $efidisk);
3265             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3266             $format = $d->{format};
3267             if ($storeid) {
3268                 $path = PVE::Storage::path($storecfg, $d->{file});
3269                 if (!defined($format)) {
3270                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3271                     $format = qemu_img_format($scfg, $volname);
3272                 }
3273             } else {
3274                 $path = $d->{file};
3275                 die "efidisk format must be specified\n"
3276                     if !defined($format);
3277             }
3278         } else {
3279             warn "no efidisk configured! Using temporary efivars disk.\n";
3280             $path = "/tmp/$vmid-ovmf.fd";
3281             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3282             $format = 'raw';
3283         }
3284
3285         my $size_str = "";
3286
3287         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3288             $size_str = ",size=" . (-s $ovmf_vars);
3289         }
3290
3291         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3292         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3293     }
3294
3295     # load q35 config
3296     if ($q35) {
3297         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3298         if (min_version($machine_version, 4, 0)) {
3299             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3300         } else {
3301             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3302         }
3303     }
3304
3305     if ($conf->{vmgenid}) {
3306         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3307     }
3308
3309     # add usb controllers
3310     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3311         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3312     push @$devices, @usbcontrollers if @usbcontrollers;
3313     my $vga = parse_vga($conf->{vga});
3314
3315     my $qxlnum = vga_conf_has_spice($conf->{vga});
3316     $vga->{type} = 'qxl' if $qxlnum;
3317
3318     if (!$vga->{type}) {
3319         if ($arch eq 'aarch64') {
3320             $vga->{type} = 'virtio';
3321         } elsif (min_version($machine_version, 2, 9)) {
3322             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3323         } else {
3324             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3325         }
3326     }
3327
3328     # enable absolute mouse coordinates (needed by vnc)
3329     my $tablet;
3330     if (defined($conf->{tablet})) {
3331         $tablet = $conf->{tablet};
3332     } else {
3333         $tablet = $defaults->{tablet};
3334         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3335         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3336     }
3337
3338     if ($tablet) {
3339         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3340         my $kbd = print_keyboarddevice_full($conf, $arch);
3341         push @$devices, '-device', $kbd if defined($kbd);
3342     }
3343
3344     my $bootorder = device_bootorder($conf);
3345
3346     # host pci device passthrough
3347     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3348         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3349
3350     # usb devices
3351     my $usb_dev_features = {};
3352     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3353
3354     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3355         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3356     push @$devices, @usbdevices if @usbdevices;
3357
3358     # serial devices
3359     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3360         if (my $path = $conf->{"serial$i"}) {
3361             if ($path eq 'socket') {
3362                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3363                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3364                 # On aarch64, serial0 is the UART device. Qemu only allows
3365                 # connecting UART devices via the '-serial' command line, as
3366                 # the device has a fixed slot on the hardware...
3367                 if ($arch eq 'aarch64' && $i == 0) {
3368                     push @$devices, '-serial', "chardev:serial$i";
3369                 } else {
3370                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3371                 }
3372             } else {
3373                 die "no such serial device\n" if ! -c $path;
3374                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3375                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3376             }
3377         }
3378     }
3379
3380     # parallel devices
3381     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3382         if (my $path = $conf->{"parallel$i"}) {
3383             die "no such parallel device\n" if ! -c $path;
3384             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3385             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3386             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3387         }
3388     }
3389
3390     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3391         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3392         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3393         push @$devices, @$audio_devs;
3394     }
3395
3396     my $sockets = 1;
3397     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3398     $sockets = $conf->{sockets} if  $conf->{sockets};
3399
3400     my $cores = $conf->{cores} || 1;
3401
3402     my $maxcpus = $sockets * $cores;
3403
3404     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3405
3406     my $allowed_vcpus = $cpuinfo->{cpus};
3407
3408     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3409         if ($allowed_vcpus < $maxcpus);
3410
3411     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3412
3413         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3414         for (my $i = 2; $i <= $vcpus; $i++)  {
3415             my $cpustr = print_cpu_device($conf,$i);
3416             push @$cmd, '-device', $cpustr;
3417         }
3418
3419     } else {
3420
3421         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3422     }
3423     push @$cmd, '-nodefaults';
3424
3425     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3426
3427     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3428
3429     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3430
3431     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3432         push @$devices, '-device', print_vga_device(
3433             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3434         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3435         push @$cmd,  '-vnc', "unix:$socket,password";
3436     } else {
3437         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3438         push @$cmd, '-nographic';
3439     }
3440
3441     # time drift fix
3442     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3443     my $useLocaltime = $conf->{localtime};
3444
3445     if ($winversion >= 5) { # windows
3446         $useLocaltime = 1 if !defined($conf->{localtime});
3447
3448         # use time drift fix when acpi is enabled
3449         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3450             $tdf = 1 if !defined($conf->{tdf});
3451         }
3452     }
3453
3454     if ($winversion >= 6) {
3455         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3456         push @$cmd, '-no-hpet';
3457     }
3458
3459     push @$rtcFlags, 'driftfix=slew' if $tdf;
3460
3461     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3462         push @$rtcFlags, "base=$conf->{startdate}";
3463     } elsif ($useLocaltime) {
3464         push @$rtcFlags, 'base=localtime';
3465     }
3466
3467     if ($forcecpu) {
3468         push @$cmd, '-cpu', $forcecpu;
3469     } else {
3470         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3471     }
3472
3473     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3474
3475     push @$cmd, '-S' if $conf->{freeze};
3476
3477     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3478
3479     my $guest_agent = parse_guest_agent($conf);
3480
3481     if ($guest_agent->{enabled}) {
3482         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3483         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3484
3485         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3486             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3487             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3488             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3489         } elsif ($guest_agent->{type} eq 'isa') {
3490             push @$devices, '-device', "isa-serial,chardev=qga0";
3491         }
3492     }
3493
3494     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3495     if ($rng && $version_guard->(4, 1, 2)) {
3496         check_rng_source($rng->{source});
3497
3498         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3499         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3500         my $limiter_str = "";
3501         if ($max_bytes) {
3502             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3503         }
3504
3505         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3506         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3507         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3508     }
3509
3510     my $spice_port;
3511
3512     if ($qxlnum) {
3513         if ($qxlnum > 1) {
3514             if ($winversion){
3515                 for (my $i = 1; $i < $qxlnum; $i++){
3516                     push @$devices, '-device', print_vga_device(
3517                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3518                 }
3519             } else {
3520                 # assume other OS works like Linux
3521                 my ($ram, $vram) = ("134217728", "67108864");
3522                 if ($vga->{memory}) {
3523                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3524                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3525                 }
3526                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3527                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3528             }
3529         }
3530
3531         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3532
3533         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3534         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3535         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3536
3537         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3538         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3539         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3540
3541         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3542         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3543
3544         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3545         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3546         if ($spice_enhancement->{foldersharing}) {
3547             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3548             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3549         }
3550
3551         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3552         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3553             if $spice_enhancement->{videostreaming};
3554
3555         push @$devices, '-spice', "$spice_opts";
3556     }
3557
3558     # enable balloon by default, unless explicitly disabled
3559     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3560         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3561         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3562     }
3563
3564     if ($conf->{watchdog}) {
3565         my $wdopts = parse_watchdog($conf->{watchdog});
3566         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3567         my $watchdog = $wdopts->{model} || 'i6300esb';
3568         push @$devices, '-device', "$watchdog$pciaddr";
3569         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3570     }
3571
3572     my $vollist = [];
3573     my $scsicontroller = {};
3574     my $ahcicontroller = {};
3575     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3576
3577     # Add iscsi initiator name if available
3578     if (my $initiator = get_initiator_name()) {
3579         push @$devices, '-iscsi', "initiator-name=$initiator";
3580     }
3581
3582     PVE::QemuConfig->foreach_volume($conf, sub {
3583         my ($ds, $drive) = @_;
3584
3585         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3586             push @$vollist, $drive->{file};
3587         }
3588
3589         # ignore efidisk here, already added in bios/fw handling code above
3590         return if $drive->{interface} eq 'efidisk';
3591
3592         $use_virtio = 1 if $ds =~ m/^virtio/;
3593
3594         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3595
3596         if ($drive->{interface} eq 'virtio'){
3597            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3598         }
3599
3600         if ($drive->{interface} eq 'scsi') {
3601
3602             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3603
3604             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3605                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3606
3607             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3608             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3609
3610             my $iothread = '';
3611             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3612                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3613                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3614             } elsif ($drive->{iothread}) {
3615                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3616             }
3617
3618             my $queues = '';
3619             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3620                 $queues = ",num_queues=$drive->{queues}";
3621             }
3622
3623             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3624                 if !$scsicontroller->{$controller};
3625             $scsicontroller->{$controller}=1;
3626         }
3627
3628         if ($drive->{interface} eq 'sata') {
3629             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3630             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3631             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3632                 if !$ahcicontroller->{$controller};
3633             $ahcicontroller->{$controller}=1;
3634         }
3635
3636         my $pbs_conf = $pbs_backing->{$ds};
3637         my $pbs_name = undef;
3638         if ($pbs_conf) {
3639             $pbs_name = "drive-$ds-pbs";
3640             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3641         }
3642
3643         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive, $pbs_name);
3644         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3645
3646         push @$devices, '-drive',$drive_cmd;
3647         push @$devices, '-device', print_drivedevice_full(
3648             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3649     });
3650
3651     for (my $i = 0; $i < $MAX_NETS; $i++) {
3652         my $netname = "net$i";
3653
3654         next if !$conf->{$netname};
3655         my $d = parse_net($conf->{$netname});
3656         next if !$d;
3657
3658         $use_virtio = 1 if $d->{model} eq 'virtio';
3659
3660         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3661
3662         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3663         push @$devices, '-netdev', $netdevfull;
3664
3665         my $netdevicefull = print_netdevice_full(
3666             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3667
3668         push @$devices, '-device', $netdevicefull;
3669     }
3670
3671     if ($conf->{ivshmem}) {
3672         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3673
3674         my $bus;
3675         if ($q35) {
3676             $bus = print_pcie_addr("ivshmem");
3677         } else {
3678             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3679         }
3680
3681         my $ivshmem_name = $ivshmem->{name} // $vmid;
3682         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3683
3684         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3685         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3686             .",size=$ivshmem->{size}M";
3687     }
3688
3689     # pci.4 is nested in pci.1
3690     $bridges->{1} = 1 if $bridges->{4};
3691
3692     if (!$q35) {
3693         # add pci bridges
3694         if (min_version($machine_version, 2, 3)) {
3695            $bridges->{1} = 1;
3696            $bridges->{2} = 1;
3697         }
3698
3699         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3700
3701     }
3702
3703     for my $k (sort {$b cmp $a} keys %$bridges) {
3704         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3705
3706         my $k_name = $k;
3707         if ($k == 2 && $legacy_igd) {
3708             $k_name = "$k-igd";
3709         }
3710         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3711
3712         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3713         if ($q35) {
3714             # add after -readconfig pve-q35.cfg
3715             splice @$devices, 2, 0, '-device', $devstr;
3716         } else {
3717             unshift @$devices, '-device', $devstr if $k > 0;
3718         }
3719     }
3720
3721     if (!$kvm) {
3722         push @$machineFlags, 'accel=tcg';
3723     }
3724
3725     my $machine_type_min = $machine_type;
3726     if ($add_pve_version) {
3727         $machine_type_min =~ s/\+pve\d+$//;
3728         $machine_type_min .= "+pve$required_pve_version";
3729     }
3730     push @$machineFlags, "type=${machine_type_min}";
3731
3732     push @$cmd, @$devices;
3733     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3734     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3735     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3736
3737     if (my $vmstate = $conf->{vmstate}) {
3738         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3739         push @$vollist, $vmstate;
3740         push @$cmd, '-loadstate', $statepath;
3741         print "activating and using '$vmstate' as vmstate\n";
3742     }
3743
3744     # add custom args
3745     if ($conf->{args}) {
3746         my $aa = PVE::Tools::split_args($conf->{args});
3747         push @$cmd, @$aa;
3748     }
3749
3750     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3751 }
3752
3753 sub check_rng_source {
3754     my ($source) = @_;
3755
3756     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3757     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3758         if ! -e $source;
3759
3760     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3761     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3762         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3763         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3764         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3765             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3766             ." to the host.\n";
3767     }
3768 }
3769
3770 sub spice_port {
3771     my ($vmid) = @_;
3772
3773     my $res = mon_cmd($vmid, 'query-spice');
3774
3775     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3776 }
3777
3778 sub vm_devices_list {
3779     my ($vmid) = @_;
3780
3781     my $res = mon_cmd($vmid, 'query-pci');
3782     my $devices_to_check = [];
3783     my $devices = {};
3784     foreach my $pcibus (@$res) {
3785         push @$devices_to_check, @{$pcibus->{devices}},
3786     }
3787
3788     while (@$devices_to_check) {
3789         my $to_check = [];
3790         for my $d (@$devices_to_check) {
3791             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3792             next if !$d->{'pci_bridge'};
3793
3794             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3795             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3796         }
3797         $devices_to_check = $to_check;
3798     }
3799
3800     my $resblock = mon_cmd($vmid, 'query-block');
3801     foreach my $block (@$resblock) {
3802         if($block->{device} =~ m/^drive-(\S+)/){
3803                 $devices->{$1} = 1;
3804         }
3805     }
3806
3807     my $resmice = mon_cmd($vmid, 'query-mice');
3808     foreach my $mice (@$resmice) {
3809         if ($mice->{name} eq 'QEMU HID Tablet') {
3810             $devices->{tablet} = 1;
3811             last;
3812         }
3813     }
3814
3815     # for usb devices there is no query-usb
3816     # but we can iterate over the entries in
3817     # qom-list path=/machine/peripheral
3818     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3819     foreach my $per (@$resperipheral) {
3820         if ($per->{name} =~ m/^usb\d+$/) {
3821             $devices->{$per->{name}} = 1;
3822         }
3823     }
3824
3825     return $devices;
3826 }
3827
3828 sub vm_deviceplug {
3829     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3830
3831     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3832
3833     my $devices_list = vm_devices_list($vmid);
3834     return 1 if defined($devices_list->{$deviceid});
3835
3836     # add PCI bridge if we need it for the device
3837     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3838
3839     if ($deviceid eq 'tablet') {
3840
3841         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3842
3843     } elsif ($deviceid eq 'keyboard') {
3844
3845         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3846
3847     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3848
3849         die "usb hotplug currently not reliable\n";
3850         # since we can't reliably hot unplug all added usb devices and usb
3851         # passthrough breaks live migration we disable usb hotplugging for now
3852         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3853
3854     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3855
3856         qemu_iothread_add($vmid, $deviceid, $device);
3857
3858         qemu_driveadd($storecfg, $vmid, $device);
3859         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3860
3861         qemu_deviceadd($vmid, $devicefull);
3862         eval { qemu_deviceaddverify($vmid, $deviceid); };
3863         if (my $err = $@) {
3864             eval { qemu_drivedel($vmid, $deviceid); };
3865             warn $@ if $@;
3866             die $err;
3867         }
3868
3869     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3870
3871
3872         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3873         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3874         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3875
3876         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3877
3878         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3879             qemu_iothread_add($vmid, $deviceid, $device);
3880             $devicefull .= ",iothread=iothread-$deviceid";
3881         }
3882
3883         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3884             $devicefull .= ",num_queues=$device->{queues}";
3885         }
3886
3887         qemu_deviceadd($vmid, $devicefull);
3888         qemu_deviceaddverify($vmid, $deviceid);
3889
3890     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3891
3892         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3893         qemu_driveadd($storecfg, $vmid, $device);
3894
3895         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3896         eval { qemu_deviceadd($vmid, $devicefull); };
3897         if (my $err = $@) {
3898             eval { qemu_drivedel($vmid, $deviceid); };
3899             warn $@ if $@;
3900             die $err;
3901         }
3902
3903     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3904
3905         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3906
3907         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3908         my $use_old_bios_files = undef;
3909         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3910
3911         my $netdevicefull = print_netdevice_full(
3912             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3913         qemu_deviceadd($vmid, $netdevicefull);
3914         eval {
3915             qemu_deviceaddverify($vmid, $deviceid);
3916             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3917         };
3918         if (my $err = $@) {
3919             eval { qemu_netdevdel($vmid, $deviceid); };
3920             warn $@ if $@;
3921             die $err;
3922         }
3923
3924     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3925
3926         my $bridgeid = $2;
3927         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3928         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3929
3930         qemu_deviceadd($vmid, $devicefull);
3931         qemu_deviceaddverify($vmid, $deviceid);
3932
3933     } else {
3934         die "can't hotplug device '$deviceid'\n";
3935     }
3936
3937     return 1;
3938 }
3939
3940 # fixme: this should raise exceptions on error!
3941 sub vm_deviceunplug {
3942     my ($vmid, $conf, $deviceid) = @_;
3943
3944     my $devices_list = vm_devices_list($vmid);
3945     return 1 if !defined($devices_list->{$deviceid});
3946
3947     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3948     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3949
3950     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3951
3952         qemu_devicedel($vmid, $deviceid);
3953
3954     } elsif ($deviceid =~ m/^usb\d+$/) {
3955
3956         die "usb hotplug currently not reliable\n";
3957         # when unplugging usb devices this way, there may be remaining usb
3958         # controllers/hubs so we disable it for now
3959         #qemu_devicedel($vmid, $deviceid);
3960         #qemu_devicedelverify($vmid, $deviceid);
3961
3962     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3963
3964         qemu_devicedel($vmid, $deviceid);
3965         qemu_devicedelverify($vmid, $deviceid);
3966         qemu_drivedel($vmid, $deviceid);
3967         qemu_iothread_del($conf, $vmid, $deviceid);
3968
3969     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3970
3971         qemu_devicedel($vmid, $deviceid);
3972         qemu_devicedelverify($vmid, $deviceid);
3973         qemu_iothread_del($conf, $vmid, $deviceid);
3974
3975     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3976
3977         qemu_devicedel($vmid, $deviceid);
3978         qemu_drivedel($vmid, $deviceid);
3979         qemu_deletescsihw($conf, $vmid, $deviceid);
3980
3981     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3982
3983         qemu_devicedel($vmid, $deviceid);
3984         qemu_devicedelverify($vmid, $deviceid);
3985         qemu_netdevdel($vmid, $deviceid);
3986
3987     } else {
3988         die "can't unplug device '$deviceid'\n";
3989     }
3990
3991     return 1;
3992 }
3993
3994 sub qemu_deviceadd {
3995     my ($vmid, $devicefull) = @_;
3996
3997     $devicefull = "driver=".$devicefull;
3998     my %options =  split(/[=,]/, $devicefull);
3999
4000     mon_cmd($vmid, "device_add" , %options);
4001 }
4002
4003 sub qemu_devicedel {
4004     my ($vmid, $deviceid) = @_;
4005
4006     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4007 }
4008
4009 sub qemu_iothread_add {
4010     my($vmid, $deviceid, $device) = @_;
4011
4012     if ($device->{iothread}) {
4013         my $iothreads = vm_iothreads_list($vmid);
4014         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4015     }
4016 }
4017
4018 sub qemu_iothread_del {
4019     my($conf, $vmid, $deviceid) = @_;
4020
4021     my $confid = $deviceid;
4022     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4023         $confid = 'scsi' . $1;
4024     }
4025     my $device = parse_drive($confid, $conf->{$confid});
4026     if ($device->{iothread}) {
4027         my $iothreads = vm_iothreads_list($vmid);
4028         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4029     }
4030 }
4031
4032 sub qemu_objectadd {
4033     my($vmid, $objectid, $qomtype) = @_;
4034
4035     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4036
4037     return 1;
4038 }
4039
4040 sub qemu_objectdel {
4041     my($vmid, $objectid) = @_;
4042
4043     mon_cmd($vmid, "object-del", id => $objectid);
4044
4045     return 1;
4046 }
4047
4048 sub qemu_driveadd {
4049     my ($storecfg, $vmid, $device) = @_;
4050
4051     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4052     $drive =~ s/\\/\\\\/g;
4053     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4054
4055     # If the command succeeds qemu prints: "OK"
4056     return 1 if $ret =~ m/OK/s;
4057
4058     die "adding drive failed: $ret\n";
4059 }
4060
4061 sub qemu_drivedel {
4062     my($vmid, $deviceid) = @_;
4063
4064     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4065     $ret =~ s/^\s+//;
4066
4067     return 1 if $ret eq "";
4068
4069     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4070     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4071
4072     die "deleting drive $deviceid failed : $ret\n";
4073 }
4074
4075 sub qemu_deviceaddverify {
4076     my ($vmid, $deviceid) = @_;
4077
4078     for (my $i = 0; $i <= 5; $i++) {
4079          my $devices_list = vm_devices_list($vmid);
4080          return 1 if defined($devices_list->{$deviceid});
4081          sleep 1;
4082     }
4083
4084     die "error on hotplug device '$deviceid'\n";
4085 }
4086
4087
4088 sub qemu_devicedelverify {
4089     my ($vmid, $deviceid) = @_;
4090
4091     # need to verify that the device is correctly removed as device_del
4092     # is async and empty return is not reliable
4093
4094     for (my $i = 0; $i <= 5; $i++) {
4095          my $devices_list = vm_devices_list($vmid);
4096          return 1 if !defined($devices_list->{$deviceid});
4097          sleep 1;
4098     }
4099
4100     die "error on hot-unplugging device '$deviceid'\n";
4101 }
4102
4103 sub qemu_findorcreatescsihw {
4104     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4105
4106     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4107
4108     my $scsihwid="$controller_prefix$controller";
4109     my $devices_list = vm_devices_list($vmid);
4110
4111     if(!defined($devices_list->{$scsihwid})) {
4112         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4113     }
4114
4115     return 1;
4116 }
4117
4118 sub qemu_deletescsihw {
4119     my ($conf, $vmid, $opt) = @_;
4120
4121     my $device = parse_drive($opt, $conf->{$opt});
4122
4123     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4124         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4125         return 1;
4126     }
4127
4128     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4129
4130     my $devices_list = vm_devices_list($vmid);
4131     foreach my $opt (keys %{$devices_list}) {
4132         if (is_valid_drivename($opt)) {
4133             my $drive = parse_drive($opt, $conf->{$opt});
4134             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4135                 return 1;
4136             }
4137         }
4138     }
4139
4140     my $scsihwid="scsihw$controller";
4141
4142     vm_deviceunplug($vmid, $conf, $scsihwid);
4143
4144     return 1;
4145 }
4146
4147 sub qemu_add_pci_bridge {
4148     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4149
4150     my $bridges = {};
4151
4152     my $bridgeid;
4153
4154     print_pci_addr($device, $bridges, $arch, $machine_type);
4155
4156     while (my ($k, $v) = each %$bridges) {
4157         $bridgeid = $k;
4158     }
4159     return 1 if !defined($bridgeid) || $bridgeid < 1;
4160
4161     my $bridge = "pci.$bridgeid";
4162     my $devices_list = vm_devices_list($vmid);
4163
4164     if (!defined($devices_list->{$bridge})) {
4165         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4166     }
4167
4168     return 1;
4169 }
4170
4171 sub qemu_set_link_status {
4172     my ($vmid, $device, $up) = @_;
4173
4174     mon_cmd($vmid, "set_link", name => $device,
4175                up => $up ? JSON::true : JSON::false);
4176 }
4177
4178 sub qemu_netdevadd {
4179     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4180
4181     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4182     my %options =  split(/[=,]/, $netdev);
4183
4184     if (defined(my $vhost = $options{vhost})) {
4185         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4186     }
4187
4188     if (defined(my $queues = $options{queues})) {
4189         $options{queues} = $queues + 0;
4190     }
4191
4192     mon_cmd($vmid, "netdev_add",  %options);
4193     return 1;
4194 }
4195
4196 sub qemu_netdevdel {
4197     my ($vmid, $deviceid) = @_;
4198
4199     mon_cmd($vmid, "netdev_del", id => $deviceid);
4200 }
4201
4202 sub qemu_usb_hotplug {
4203     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4204
4205     return if !$device;
4206
4207     # remove the old one first
4208     vm_deviceunplug($vmid, $conf, $deviceid);
4209
4210     # check if xhci controller is necessary and available
4211     if ($device->{usb3}) {
4212
4213         my $devicelist = vm_devices_list($vmid);
4214
4215         if (!$devicelist->{xhci}) {
4216             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4217             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4218         }
4219     }
4220     my $d = parse_usb_device($device->{host});
4221     $d->{usb3} = $device->{usb3};
4222
4223     # add the new one
4224     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4225 }
4226
4227 sub qemu_cpu_hotplug {
4228     my ($vmid, $conf, $vcpus) = @_;
4229
4230     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4231
4232     my $sockets = 1;
4233     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4234     $sockets = $conf->{sockets} if  $conf->{sockets};
4235     my $cores = $conf->{cores} || 1;
4236     my $maxcpus = $sockets * $cores;
4237
4238     $vcpus = $maxcpus if !$vcpus;
4239
4240     die "you can't add more vcpus than maxcpus\n"
4241         if $vcpus > $maxcpus;
4242
4243     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4244
4245     if ($vcpus < $currentvcpus) {
4246
4247         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4248
4249             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4250                 qemu_devicedel($vmid, "cpu$i");
4251                 my $retry = 0;
4252                 my $currentrunningvcpus = undef;
4253                 while (1) {
4254                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4255                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4256                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4257                     $retry++;
4258                     sleep 1;
4259                 }
4260                 #update conf after each succesfull cpu unplug
4261                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4262                 PVE::QemuConfig->write_config($vmid, $conf);
4263             }
4264         } else {
4265             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4266         }
4267
4268         return;
4269     }
4270
4271     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4272     die "vcpus in running vm does not match its configuration\n"
4273         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4274
4275     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4276
4277         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4278             my $cpustr = print_cpu_device($conf, $i);
4279             qemu_deviceadd($vmid, $cpustr);
4280
4281             my $retry = 0;
4282             my $currentrunningvcpus = undef;
4283             while (1) {
4284                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4285                 last if scalar(@{$currentrunningvcpus}) == $i;
4286                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4287                 sleep 1;
4288                 $retry++;
4289             }
4290             #update conf after each succesfull cpu hotplug
4291             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4292             PVE::QemuConfig->write_config($vmid, $conf);
4293         }
4294     } else {
4295
4296         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4297             mon_cmd($vmid, "cpu-add", id => int($i));
4298         }
4299     }
4300 }
4301
4302 sub qemu_block_set_io_throttle {
4303     my ($vmid, $deviceid,
4304         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4305         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4306         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4307         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4308
4309     return if !check_running($vmid) ;
4310
4311     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4312         bps => int($bps),
4313         bps_rd => int($bps_rd),
4314         bps_wr => int($bps_wr),
4315         iops => int($iops),
4316         iops_rd => int($iops_rd),
4317         iops_wr => int($iops_wr),
4318         bps_max => int($bps_max),
4319         bps_rd_max => int($bps_rd_max),
4320         bps_wr_max => int($bps_wr_max),
4321         iops_max => int($iops_max),
4322         iops_rd_max => int($iops_rd_max),
4323         iops_wr_max => int($iops_wr_max),
4324         bps_max_length => int($bps_max_length),
4325         bps_rd_max_length => int($bps_rd_max_length),
4326         bps_wr_max_length => int($bps_wr_max_length),
4327         iops_max_length => int($iops_max_length),
4328         iops_rd_max_length => int($iops_rd_max_length),
4329         iops_wr_max_length => int($iops_wr_max_length),
4330     );
4331
4332 }
4333
4334 sub qemu_block_resize {
4335     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4336
4337     my $running = check_running($vmid);
4338
4339     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4340
4341     return if !$running;
4342
4343     my $padding = (1024 - $size % 1024) % 1024;
4344     $size = $size + $padding;
4345
4346     mon_cmd(
4347         $vmid,
4348         "block_resize",
4349         device => $deviceid,
4350         size => int($size),
4351         timeout => 60,
4352     );
4353 }
4354
4355 sub qemu_volume_snapshot {
4356     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4357
4358     my $running = check_running($vmid);
4359
4360     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4361         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4362     } else {
4363         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4364     }
4365 }
4366
4367 sub qemu_volume_snapshot_delete {
4368     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4369
4370     my $running = check_running($vmid);
4371
4372     if($running) {
4373
4374         $running = undef;
4375         my $conf = PVE::QemuConfig->load_config($vmid);
4376         PVE::QemuConfig->foreach_volume($conf, sub {
4377             my ($ds, $drive) = @_;
4378             $running = 1 if $drive->{file} eq $volid;
4379         });
4380     }
4381
4382     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4383         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4384     } else {
4385         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4386     }
4387 }
4388
4389 sub set_migration_caps {
4390     my ($vmid, $savevm) = @_;
4391
4392     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4393
4394     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4395     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4396
4397     my $cap_ref = [];
4398
4399     my $enabled_cap = {
4400         "auto-converge" => 1,
4401         "xbzrle" => 1,
4402         "x-rdma-pin-all" => 0,
4403         "zero-blocks" => 0,
4404         "compress" => 0,
4405         "dirty-bitmaps" => $dirty_bitmaps,
4406     };
4407
4408     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4409
4410     for my $supported_capability (@$supported_capabilities) {
4411         push @$cap_ref, {
4412             capability => $supported_capability->{capability},
4413             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4414         };
4415     }
4416
4417     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4418 }
4419
4420 sub foreach_volid {
4421     my ($conf, $func, @param) = @_;
4422
4423     my $volhash = {};
4424
4425     my $test_volid = sub {
4426         my ($key, $drive, $snapname) = @_;
4427
4428         my $volid = $drive->{file};
4429         return if !$volid;
4430
4431         $volhash->{$volid}->{cdrom} //= 1;
4432         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4433
4434         my $replicate = $drive->{replicate} // 1;
4435         $volhash->{$volid}->{replicate} //= 0;
4436         $volhash->{$volid}->{replicate} = 1 if $replicate;
4437
4438         $volhash->{$volid}->{shared} //= 0;
4439         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4440
4441         $volhash->{$volid}->{referenced_in_config} //= 0;
4442         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4443
4444         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4445             if defined($snapname);
4446
4447         my $size = $drive->{size};
4448         $volhash->{$volid}->{size} //= $size if $size;
4449
4450         $volhash->{$volid}->{is_vmstate} //= 0;
4451         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4452
4453         $volhash->{$volid}->{is_unused} //= 0;
4454         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4455     };
4456
4457     my $include_opts = {
4458         extra_keys => ['vmstate'],
4459         include_unused => 1,
4460     };
4461
4462     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4463     foreach my $snapname (keys %{$conf->{snapshots}}) {
4464         my $snap = $conf->{snapshots}->{$snapname};
4465         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4466     }
4467
4468     foreach my $volid (keys %$volhash) {
4469         &$func($volid, $volhash->{$volid}, @param);
4470     }
4471 }
4472
4473 my $fast_plug_option = {
4474     'lock' => 1,
4475     'name' => 1,
4476     'onboot' => 1,
4477     'shares' => 1,
4478     'startup' => 1,
4479     'description' => 1,
4480     'protection' => 1,
4481     'vmstatestorage' => 1,
4482     'hookscript' => 1,
4483     'tags' => 1,
4484 };
4485
4486 # hotplug changes in [PENDING]
4487 # $selection hash can be used to only apply specified options, for
4488 # example: { cores => 1 } (only apply changed 'cores')
4489 # $errors ref is used to return error messages
4490 sub vmconfig_hotplug_pending {
4491     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4492
4493     my $defaults = load_defaults();
4494     my $arch = get_vm_arch($conf);
4495     my $machine_type = get_vm_machine($conf, undef, $arch);
4496
4497     # commit values which do not have any impact on running VM first
4498     # Note: those option cannot raise errors, we we do not care about
4499     # $selection and always apply them.
4500
4501     my $add_error = sub {
4502         my ($opt, $msg) = @_;
4503         $errors->{$opt} = "hotplug problem - $msg";
4504     };
4505
4506     my $changes = 0;
4507     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4508         if ($fast_plug_option->{$opt}) {
4509             $conf->{$opt} = $conf->{pending}->{$opt};
4510             delete $conf->{pending}->{$opt};
4511             $changes = 1;
4512         }
4513     }
4514
4515     if ($changes) {
4516         PVE::QemuConfig->write_config($vmid, $conf);
4517     }
4518
4519     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4520
4521     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4522     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4523     foreach my $opt (sort keys %$pending_delete_hash) {
4524         next if $selection && !$selection->{$opt};
4525         my $force = $pending_delete_hash->{$opt}->{force};
4526         eval {
4527             if ($opt eq 'hotplug') {
4528                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4529             } elsif ($opt eq 'tablet') {
4530                 die "skip\n" if !$hotplug_features->{usb};
4531                 if ($defaults->{tablet}) {
4532                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4533                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4534                         if $arch eq 'aarch64';
4535                 } else {
4536                     vm_deviceunplug($vmid, $conf, 'tablet');
4537                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4538                 }
4539             } elsif ($opt =~ m/^usb\d+/) {
4540                 die "skip\n";
4541                 # since we cannot reliably hot unplug usb devices we are disabling it
4542                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4543                 #vm_deviceunplug($vmid, $conf, $opt);
4544             } elsif ($opt eq 'vcpus') {
4545                 die "skip\n" if !$hotplug_features->{cpu};
4546                 qemu_cpu_hotplug($vmid, $conf, undef);
4547             } elsif ($opt eq 'balloon') {
4548                 # enable balloon device is not hotpluggable
4549                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4550                 # here we reset the ballooning value to memory
4551                 my $balloon = $conf->{memory} || $defaults->{memory};
4552                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4553             } elsif ($fast_plug_option->{$opt}) {
4554                 # do nothing
4555             } elsif ($opt =~ m/^net(\d+)$/) {
4556                 die "skip\n" if !$hotplug_features->{network};
4557                 vm_deviceunplug($vmid, $conf, $opt);
4558             } elsif (is_valid_drivename($opt)) {
4559                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4560                 vm_deviceunplug($vmid, $conf, $opt);
4561                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4562             } elsif ($opt =~ m/^memory$/) {
4563                 die "skip\n" if !$hotplug_features->{memory};
4564                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4565             } elsif ($opt eq 'cpuunits') {
4566                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4567             } elsif ($opt eq 'cpulimit') {
4568                 $cgroup->change_cpu_quota(-1, 100000);
4569             } else {
4570                 die "skip\n";
4571             }
4572         };
4573         if (my $err = $@) {
4574             &$add_error($opt, $err) if $err ne "skip\n";
4575         } else {
4576             delete $conf->{$opt};
4577             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4578         }
4579     }
4580
4581     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4582     $apply_pending_cloudinit = sub {
4583         return if $apply_pending_cloudinit_done; # once is enough
4584         $apply_pending_cloudinit_done = 1; # once is enough
4585
4586         my ($key, $value) = @_;
4587
4588         my @cloudinit_opts = keys %$confdesc_cloudinit;
4589         foreach my $opt (keys %{$conf->{pending}}) {
4590             next if !grep { $_ eq $opt } @cloudinit_opts;
4591             $conf->{$opt} = delete $conf->{pending}->{$opt};
4592         }
4593
4594         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4595         foreach my $opt (sort keys %$pending_delete_hash) {
4596             next if !grep { $_ eq $opt } @cloudinit_opts;
4597             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4598             delete $conf->{$opt};
4599         }
4600
4601         my $new_conf = { %$conf };
4602         $new_conf->{$key} = $value;
4603         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4604     };
4605
4606     foreach my $opt (keys %{$conf->{pending}}) {
4607         next if $selection && !$selection->{$opt};
4608         my $value = $conf->{pending}->{$opt};
4609         eval {
4610             if ($opt eq 'hotplug') {
4611                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4612             } elsif ($opt eq 'tablet') {
4613                 die "skip\n" if !$hotplug_features->{usb};
4614                 if ($value == 1) {
4615                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4616                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4617                         if $arch eq 'aarch64';
4618                 } elsif ($value == 0) {
4619                     vm_deviceunplug($vmid, $conf, 'tablet');
4620                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4621                 }
4622             } elsif ($opt =~ m/^usb\d+$/) {
4623                 die "skip\n";
4624                 # since we cannot reliably hot unplug usb devices we disable it for now
4625                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4626                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4627                 #die "skip\n" if !$d;
4628                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4629             } elsif ($opt eq 'vcpus') {
4630                 die "skip\n" if !$hotplug_features->{cpu};
4631                 qemu_cpu_hotplug($vmid, $conf, $value);
4632             } elsif ($opt eq 'balloon') {
4633                 # enable/disable balloning device is not hotpluggable
4634                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4635                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4636                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4637
4638                 # allow manual ballooning if shares is set to zero
4639                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4640                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4641                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4642                 }
4643             } elsif ($opt =~ m/^net(\d+)$/) {
4644                 # some changes can be done without hotplug
4645                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4646                                     $vmid, $opt, $value, $arch, $machine_type);
4647             } elsif (is_valid_drivename($opt)) {
4648                 die "skip\n" if $opt eq 'efidisk0';
4649                 # some changes can be done without hotplug
4650                 my $drive = parse_drive($opt, $value);
4651                 if (drive_is_cloudinit($drive)) {
4652                     &$apply_pending_cloudinit($opt, $value);
4653                 }
4654                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4655                                      $vmid, $opt, $value, $arch, $machine_type);
4656             } elsif ($opt =~ m/^memory$/) { #dimms
4657                 die "skip\n" if !$hotplug_features->{memory};
4658                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4659             } elsif ($opt eq 'cpuunits') {
4660                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4661             } elsif ($opt eq 'cpulimit') {
4662                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4663                 $cgroup->change_cpu_quota($cpulimit, 100000);
4664             } else {
4665                 die "skip\n";  # skip non-hot-pluggable options
4666             }
4667         };
4668         if (my $err = $@) {
4669             &$add_error($opt, $err) if $err ne "skip\n";
4670         } else {
4671             $conf->{$opt} = $value;
4672             delete $conf->{pending}->{$opt};
4673         }
4674     }
4675
4676     PVE::QemuConfig->write_config($vmid, $conf);
4677 }
4678
4679 sub try_deallocate_drive {
4680     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4681
4682     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4683         my $volid = $drive->{file};
4684         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4685             my $sid = PVE::Storage::parse_volume_id($volid);
4686             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4687
4688             # check if the disk is really unused
4689             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4690                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4691             PVE::Storage::vdisk_free($storecfg, $volid);
4692             return 1;
4693         } else {
4694             # If vm is not owner of this disk remove from config
4695             return 1;
4696         }
4697     }
4698
4699     return;
4700 }
4701
4702 sub vmconfig_delete_or_detach_drive {
4703     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4704
4705     my $drive = parse_drive($opt, $conf->{$opt});
4706
4707     my $rpcenv = PVE::RPCEnvironment::get();
4708     my $authuser = $rpcenv->get_user();
4709
4710     if ($force) {
4711         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4712         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4713     } else {
4714         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4715     }
4716 }
4717
4718
4719
4720 sub vmconfig_apply_pending {
4721     my ($vmid, $conf, $storecfg, $errors) = @_;
4722
4723     my $add_apply_error = sub {
4724         my ($opt, $msg) = @_;
4725         my $err_msg = "unable to apply pending change $opt : $msg";
4726         $errors->{$opt} = $err_msg;
4727         warn $err_msg;
4728     };
4729
4730     # cold plug
4731
4732     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4733     foreach my $opt (sort keys %$pending_delete_hash) {
4734         my $force = $pending_delete_hash->{$opt}->{force};
4735         eval {
4736             if ($opt =~ m/^unused/) {
4737                 die "internal error";
4738             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4739                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4740             }
4741         };
4742         if (my $err = $@) {
4743             $add_apply_error->($opt, $err);
4744         } else {
4745             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4746             delete $conf->{$opt};
4747         }
4748     }
4749
4750     PVE::QemuConfig->cleanup_pending($conf);
4751
4752     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4753         next if $opt eq 'delete'; # just to be sure
4754         eval {
4755             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4756                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4757             }
4758         };
4759         if (my $err = $@) {
4760             $add_apply_error->($opt, $err);
4761         } else {
4762             $conf->{$opt} = delete $conf->{pending}->{$opt};
4763         }
4764     }
4765
4766     # write all changes at once to avoid unnecessary i/o
4767     PVE::QemuConfig->write_config($vmid, $conf);
4768 }
4769
4770 sub vmconfig_update_net {
4771     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4772
4773     my $newnet = parse_net($value);
4774
4775     if ($conf->{$opt}) {
4776         my $oldnet = parse_net($conf->{$opt});
4777
4778         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4779             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4780             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4781             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4782
4783             # for non online change, we try to hot-unplug
4784             die "skip\n" if !$hotplug;
4785             vm_deviceunplug($vmid, $conf, $opt);
4786         } else {
4787
4788             die "internal error" if $opt !~ m/net(\d+)/;
4789             my $iface = "tap${vmid}i$1";
4790
4791             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4792                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4793                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4794                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4795                 PVE::Network::tap_unplug($iface);
4796
4797                 if ($have_sdn) {
4798                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4799                 } else {
4800                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4801                 }
4802             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4803                 # Rate can be applied on its own but any change above needs to
4804                 # include the rate in tap_plug since OVS resets everything.
4805                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4806             }
4807
4808             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4809                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4810             }
4811
4812             return 1;
4813         }
4814     }
4815
4816     if ($hotplug) {
4817         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4818     } else {
4819         die "skip\n";
4820     }
4821 }
4822
4823 sub vmconfig_update_disk {
4824     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4825
4826     my $drive = parse_drive($opt, $value);
4827
4828     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4829         my $media = $drive->{media} || 'disk';
4830         my $oldmedia = $old_drive->{media} || 'disk';
4831         die "unable to change media type\n" if $media ne $oldmedia;
4832
4833         if (!drive_is_cdrom($old_drive)) {
4834
4835             if ($drive->{file} ne $old_drive->{file}) {
4836
4837                 die "skip\n" if !$hotplug;
4838
4839                 # unplug and register as unused
4840                 vm_deviceunplug($vmid, $conf, $opt);
4841                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4842
4843             } else {
4844                 # update existing disk
4845
4846                 # skip non hotpluggable value
4847                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4848                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4849                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4850                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4851                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4852                     die "skip\n";
4853                 }
4854
4855                 # apply throttle
4856                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4857                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4858                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4859                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4860                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4861                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4862                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4863                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4864                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4865                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4866                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4867                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4868                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4869                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4870                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4871                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4872                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4873                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4874
4875                     qemu_block_set_io_throttle(
4876                         $vmid,"drive-$opt",
4877                         ($drive->{mbps} || 0)*1024*1024,
4878                         ($drive->{mbps_rd} || 0)*1024*1024,
4879                         ($drive->{mbps_wr} || 0)*1024*1024,
4880                          $drive->{iops} || 0,
4881                          $drive->{iops_rd} || 0,
4882                          $drive->{iops_wr} || 0,
4883                         ($drive->{mbps_max} || 0)*1024*1024,
4884                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4885                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4886                          $drive->{iops_max} || 0,
4887                          $drive->{iops_rd_max} || 0,
4888                          $drive->{iops_wr_max} || 0,
4889                          $drive->{bps_max_length} || 1,
4890                          $drive->{bps_rd_max_length} || 1,
4891                          $drive->{bps_wr_max_length} || 1,
4892                          $drive->{iops_max_length} || 1,
4893                          $drive->{iops_rd_max_length} || 1,
4894                          $drive->{iops_wr_max_length} || 1,
4895                     );
4896
4897                 }
4898
4899                 return 1;
4900             }
4901
4902         } else { # cdrom
4903
4904             if ($drive->{file} eq 'none') {
4905                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4906                 if (drive_is_cloudinit($old_drive)) {
4907                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4908                 }
4909             } else {
4910                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4911
4912                 # force eject if locked
4913                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4914
4915                 if ($path) {
4916                     mon_cmd($vmid, "blockdev-change-medium",
4917                         id => "$opt", filename => "$path");
4918                 }
4919             }
4920
4921             return 1;
4922         }
4923     }
4924
4925     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4926     # hotplug new disks
4927     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4928     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4929 }
4930
4931 # called in locked context by incoming migration
4932 sub vm_migrate_get_nbd_disks {
4933     my ($storecfg, $conf, $replicated_volumes) = @_;
4934
4935     my $local_volumes = {};
4936     PVE::QemuConfig->foreach_volume($conf, sub {
4937         my ($ds, $drive) = @_;
4938
4939         return if drive_is_cdrom($drive);
4940
4941         my $volid = $drive->{file};
4942
4943         return if !$volid;
4944
4945         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4946
4947         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4948         return if $scfg->{shared};
4949
4950         # replicated disks re-use existing state via bitmap
4951         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4952         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4953     });
4954     return $local_volumes;
4955 }
4956
4957 # called in locked context by incoming migration
4958 sub vm_migrate_alloc_nbd_disks {
4959     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4960
4961     my $format = undef;
4962
4963     my $nbd = {};
4964     foreach my $opt (sort keys %$source_volumes) {
4965         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4966
4967         if ($use_existing) {
4968             $nbd->{$opt}->{drivestr} = print_drive($drive);
4969             $nbd->{$opt}->{volid} = $volid;
4970             $nbd->{$opt}->{replicated} = 1;
4971             next;
4972         }
4973
4974         # If a remote storage is specified and the format of the original
4975         # volume is not available there, fall back to the default format.
4976         # Otherwise use the same format as the original.
4977         if (!$storagemap->{identity}) {
4978             $storeid = map_storage($storagemap, $storeid);
4979             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4980             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4981             my $fileFormat = qemu_img_format($scfg, $volname);
4982             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4983         } else {
4984             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4985             $format = qemu_img_format($scfg, $volname);
4986         }
4987
4988         my $size = $drive->{size} / 1024;
4989         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4990         my $newdrive = $drive;
4991         $newdrive->{format} = $format;
4992         $newdrive->{file} = $newvolid;
4993         my $drivestr = print_drive($newdrive);
4994         $nbd->{$opt}->{drivestr} = $drivestr;
4995         $nbd->{$opt}->{volid} = $newvolid;
4996     }
4997
4998     return $nbd;
4999 }
5000
5001 # see vm_start_nolock for parameters, additionally:
5002 # migrate_opts:
5003 #   storagemap = parsed storage map for allocating NBD disks
5004 sub vm_start {
5005     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5006
5007     return PVE::QemuConfig->lock_config($vmid, sub {
5008         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5009
5010         die "you can't start a vm if it's a template\n"
5011             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5012
5013         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5014         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5015
5016         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5017
5018         if ($has_backup_lock && $running) {
5019             # a backup is currently running, attempt to start the guest in the
5020             # existing QEMU instance
5021             return vm_resume($vmid);
5022         }
5023
5024         PVE::QemuConfig->check_lock($conf)
5025             if !($params->{skiplock} || $has_suspended_lock);
5026
5027         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5028
5029         die "VM $vmid already running\n" if $running;
5030
5031         if (my $storagemap = $migrate_opts->{storagemap}) {
5032             my $replicated = $migrate_opts->{replicated_volumes};
5033             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5034             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5035
5036             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5037                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5038             }
5039         }
5040
5041         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5042     });
5043 }
5044
5045
5046 # params:
5047 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5048 #   skiplock => 0/1, skip checking for config lock
5049 #   skiptemplate => 0/1, skip checking whether VM is template
5050 #   forcemachine => to force Qemu machine (rollback/migration)
5051 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5052 #   timeout => in seconds
5053 #   paused => start VM in paused state (backup)
5054 #   resume => resume from hibernation
5055 #   pbs-backing => {
5056 #      sata0 => {
5057 #         repository
5058 #         snapshot
5059 #         keyfile
5060 #         archive
5061 #      },
5062 #      virtio2 => ...
5063 #   }
5064 # migrate_opts:
5065 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5066 #   migratedfrom => source node
5067 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5068 #   network => CIDR of migration network
5069 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5070 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5071 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5072 sub vm_start_nolock {
5073     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5074
5075     my $statefile = $params->{statefile};
5076     my $resume = $params->{resume};
5077
5078     my $migratedfrom = $migrate_opts->{migratedfrom};
5079     my $migration_type = $migrate_opts->{type};
5080
5081     my $res = {};
5082
5083     # clean up leftover reboot request files
5084     eval { clear_reboot_request($vmid); };
5085     warn $@ if $@;
5086
5087     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5088         vmconfig_apply_pending($vmid, $conf, $storecfg);
5089         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5090     }
5091
5092     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5093
5094     my $defaults = load_defaults();
5095
5096     # set environment variable useful inside network script
5097     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5098
5099     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5100
5101     my $forcemachine = $params->{forcemachine};
5102     my $forcecpu = $params->{forcecpu};
5103     if ($resume) {
5104         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5105         $forcemachine = $conf->{runningmachine};
5106         $forcecpu = $conf->{runningcpu};
5107         print "Resuming suspended VM\n";
5108     }
5109
5110     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5111         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5112
5113     my $migration_ip;
5114     my $get_migration_ip = sub {
5115         my ($nodename) = @_;
5116
5117         return $migration_ip if defined($migration_ip);
5118
5119         my $cidr = $migrate_opts->{network};
5120
5121         if (!defined($cidr)) {
5122             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5123             $cidr = $dc_conf->{migration}->{network};
5124         }
5125
5126         if (defined($cidr)) {
5127             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5128
5129             die "could not get IP: no address configured on local " .
5130                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5131
5132             die "could not get IP: multiple addresses configured on local " .
5133                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5134
5135             $migration_ip = @$ips[0];
5136         }
5137
5138         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5139             if !defined($migration_ip);
5140
5141         return $migration_ip;
5142     };
5143
5144     my $migrate_uri;
5145     if ($statefile) {
5146         if ($statefile eq 'tcp') {
5147             my $localip = "localhost";
5148             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5149             my $nodename = nodename();
5150
5151             if (!defined($migration_type)) {
5152                 if (defined($datacenterconf->{migration}->{type})) {
5153                     $migration_type = $datacenterconf->{migration}->{type};
5154                 } else {
5155                     $migration_type = 'secure';
5156                 }
5157             }
5158
5159             if ($migration_type eq 'insecure') {
5160                 $localip = $get_migration_ip->($nodename);
5161                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5162             }
5163
5164             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5165             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5166             $migrate_uri = "tcp:${localip}:${migrate_port}";
5167             push @$cmd, '-incoming', $migrate_uri;
5168             push @$cmd, '-S';
5169
5170         } elsif ($statefile eq 'unix') {
5171             # should be default for secure migrations as a ssh TCP forward
5172             # tunnel is not deterministic reliable ready and fails regurarly
5173             # to set up in time, so use UNIX socket forwards
5174             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5175             unlink $socket_addr;
5176
5177             $migrate_uri = "unix:$socket_addr";
5178
5179             push @$cmd, '-incoming', $migrate_uri;
5180             push @$cmd, '-S';
5181
5182         } elsif (-e $statefile) {
5183             push @$cmd, '-loadstate', $statefile;
5184         } else {
5185             my $statepath = PVE::Storage::path($storecfg, $statefile);
5186             push @$vollist, $statefile;
5187             push @$cmd, '-loadstate', $statepath;
5188         }
5189     } elsif ($params->{paused}) {
5190         push @$cmd, '-S';
5191     }
5192
5193     # host pci devices
5194     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5195       my $d = parse_hostpci($conf->{"hostpci$i"});
5196       next if !$d;
5197       my $pcidevices = $d->{pciid};
5198       foreach my $pcidevice (@$pcidevices) {
5199             my $pciid = $pcidevice->{id};
5200
5201             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5202             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5203             die "no pci device info for device '$pciid'\n" if !$info;
5204
5205             if ($d->{mdev}) {
5206                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5207                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5208             } else {
5209                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5210                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5211                 die "can't reset PCI device '$pciid'\n"
5212                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5213             }
5214       }
5215     }
5216
5217     PVE::Storage::activate_volumes($storecfg, $vollist);
5218
5219     eval {
5220         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5221             outfunc => sub {}, errfunc => sub {});
5222     };
5223     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5224     # timeout should be more than enough here...
5225     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5226
5227     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5228                                               : $defaults->{cpuunits};
5229
5230     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5231     my %run_params = (
5232         timeout => $statefile ? undef : $start_timeout,
5233         umask => 0077,
5234         noerr => 1,
5235     );
5236
5237     # when migrating, prefix QEMU output so other side can pick up any
5238     # errors that might occur and show the user
5239     if ($migratedfrom) {
5240         $run_params{quiet} = 1;
5241         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5242     }
5243
5244     my %properties = (
5245         Slice => 'qemu.slice',
5246         KillMode => 'none'
5247     );
5248
5249     if (PVE::CGroup::cgroup_mode() == 2) {
5250         $properties{CPUWeight} = $cpuunits;
5251     } else {
5252         $properties{CPUShares} = $cpuunits;
5253     }
5254
5255     if (my $cpulimit = $conf->{cpulimit}) {
5256         $properties{CPUQuota} = int($cpulimit * 100);
5257     }
5258     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5259
5260     my $run_qemu = sub {
5261         PVE::Tools::run_fork sub {
5262             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5263
5264             my $exitcode = run_command($cmd, %run_params);
5265             die "QEMU exited with code $exitcode\n" if $exitcode;
5266         };
5267     };
5268
5269     if ($conf->{hugepages}) {
5270
5271         my $code = sub {
5272             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5273             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5274
5275             PVE::QemuServer::Memory::hugepages_mount();
5276             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5277
5278             eval { $run_qemu->() };
5279             if (my $err = $@) {
5280                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5281                     if !$conf->{keephugepages};
5282                 die $err;
5283             }
5284
5285             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5286                 if !$conf->{keephugepages};
5287         };
5288         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5289
5290     } else {
5291         eval { $run_qemu->() };
5292     }
5293
5294     if (my $err = $@) {
5295         # deactivate volumes if start fails
5296         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5297         die "start failed: $err";
5298     }
5299
5300     print "migration listens on $migrate_uri\n" if $migrate_uri;
5301     $res->{migrate_uri} = $migrate_uri;
5302
5303     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5304         eval { mon_cmd($vmid, "cont"); };
5305         warn $@ if $@;
5306     }
5307
5308     #start nbd server for storage migration
5309     if (my $nbd = $migrate_opts->{nbd}) {
5310         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5311
5312         my $migrate_storage_uri;
5313         # nbd_protocol_version > 0 for unix socket support
5314         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5315             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5316             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5317             $migrate_storage_uri = "nbd:unix:$socket_path";
5318         } else {
5319             my $nodename = nodename();
5320             my $localip = $get_migration_ip->($nodename);
5321             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5322             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5323
5324             mon_cmd($vmid, "nbd-server-start", addr => {
5325                 type => 'inet',
5326                 data => {
5327                     host => "${localip}",
5328                     port => "${storage_migrate_port}",
5329                 },
5330             });
5331             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5332             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5333         }
5334
5335         $res->{migrate_storage_uri} = $migrate_storage_uri;
5336
5337         foreach my $opt (sort keys %$nbd) {
5338             my $drivestr = $nbd->{$opt}->{drivestr};
5339             my $volid = $nbd->{$opt}->{volid};
5340             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5341             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5342             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5343             print "re-using replicated volume: $opt - $volid\n"
5344                 if $nbd->{$opt}->{replicated};
5345
5346             $res->{drives}->{$opt} = $nbd->{$opt};
5347             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5348         }
5349     }
5350
5351     if ($migratedfrom) {
5352         eval {
5353             set_migration_caps($vmid);
5354         };
5355         warn $@ if $@;
5356
5357         if ($spice_port) {
5358             print "spice listens on port $spice_port\n";
5359             $res->{spice_port} = $spice_port;
5360             if ($migrate_opts->{spice_ticket}) {
5361                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5362                     $migrate_opts->{spice_ticket});
5363                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5364             }
5365         }
5366
5367     } else {
5368         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5369             if !$statefile && $conf->{balloon};
5370
5371         foreach my $opt (keys %$conf) {
5372             next if $opt !~  m/^net\d+$/;
5373             my $nicconf = parse_net($conf->{$opt});
5374             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5375         }
5376     }
5377
5378     mon_cmd($vmid, 'qom-set',
5379                 path => "machine/peripheral/balloon0",
5380                 property => "guest-stats-polling-interval",
5381                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5382
5383     if ($resume) {
5384         print "Resumed VM, removing state\n";
5385         if (my $vmstate = $conf->{vmstate}) {
5386             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5387             PVE::Storage::vdisk_free($storecfg, $vmstate);
5388         }
5389         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5390         PVE::QemuConfig->write_config($vmid, $conf);
5391     }
5392
5393     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5394
5395     return $res;
5396 }
5397
5398 sub vm_commandline {
5399     my ($storecfg, $vmid, $snapname) = @_;
5400
5401     my $conf = PVE::QemuConfig->load_config($vmid);
5402     my $forcemachine;
5403     my $forcecpu;
5404
5405     if ($snapname) {
5406         my $snapshot = $conf->{snapshots}->{$snapname};
5407         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5408
5409         # check for machine or CPU overrides in snapshot
5410         $forcemachine = $snapshot->{runningmachine};
5411         $forcecpu = $snapshot->{runningcpu};
5412
5413         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5414
5415         $conf = $snapshot;
5416     }
5417
5418     my $defaults = load_defaults();
5419
5420     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5421         $forcemachine, $forcecpu);
5422
5423     return PVE::Tools::cmd2string($cmd);
5424 }
5425
5426 sub vm_reset {
5427     my ($vmid, $skiplock) = @_;
5428
5429     PVE::QemuConfig->lock_config($vmid, sub {
5430
5431         my $conf = PVE::QemuConfig->load_config($vmid);
5432
5433         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5434
5435         mon_cmd($vmid, "system_reset");
5436     });
5437 }
5438
5439 sub get_vm_volumes {
5440     my ($conf) = @_;
5441
5442     my $vollist = [];
5443     foreach_volid($conf, sub {
5444         my ($volid, $attr) = @_;
5445
5446         return if $volid =~ m|^/|;
5447
5448         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5449         return if !$sid;
5450
5451         push @$vollist, $volid;
5452     });
5453
5454     return $vollist;
5455 }
5456
5457 sub vm_stop_cleanup {
5458     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5459
5460     eval {
5461
5462         if (!$keepActive) {
5463             my $vollist = get_vm_volumes($conf);
5464             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5465         }
5466
5467         foreach my $ext (qw(mon qmp pid vnc qga)) {
5468             unlink "/var/run/qemu-server/${vmid}.$ext";
5469         }
5470
5471         if ($conf->{ivshmem}) {
5472             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5473             # just delete it for now, VMs which have this already open do not
5474             # are affected, but new VMs will get a separated one. If this
5475             # becomes an issue we either add some sort of ref-counting or just
5476             # add a "don't delete on stop" flag to the ivshmem format.
5477             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5478         }
5479
5480         foreach my $key (keys %$conf) {
5481             next if $key !~ m/^hostpci(\d+)$/;
5482             my $hostpciindex = $1;
5483             my $d = parse_hostpci($conf->{$key});
5484             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5485
5486             foreach my $pci (@{$d->{pciid}}) {
5487                 my $pciid = $pci->{id};
5488                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5489             }
5490         }
5491
5492         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5493     };
5494     warn $@ if $@; # avoid errors - just warn
5495 }
5496
5497 # call only in locked context
5498 sub _do_vm_stop {
5499     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5500
5501     my $pid = check_running($vmid, $nocheck);
5502     return if !$pid;
5503
5504     my $conf;
5505     if (!$nocheck) {
5506         $conf = PVE::QemuConfig->load_config($vmid);
5507         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5508         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5509             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5510             $timeout = $opts->{down} if $opts->{down};
5511         }
5512         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5513     }
5514
5515     eval {
5516         if ($shutdown) {
5517             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5518                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5519             } else {
5520                 mon_cmd($vmid, "system_powerdown");
5521             }
5522         } else {
5523             mon_cmd($vmid, "quit");
5524         }
5525     };
5526     my $err = $@;
5527
5528     if (!$err) {
5529         $timeout = 60 if !defined($timeout);
5530
5531         my $count = 0;
5532         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5533             $count++;
5534             sleep 1;
5535         }
5536
5537         if ($count >= $timeout) {
5538             if ($force) {
5539                 warn "VM still running - terminating now with SIGTERM\n";
5540                 kill 15, $pid;
5541             } else {
5542                 die "VM quit/powerdown failed - got timeout\n";
5543             }
5544         } else {
5545             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5546             return;
5547         }
5548     } else {
5549        if (!check_running($vmid, $nocheck)) {
5550             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5551             return;
5552        }
5553        if ($force) {
5554             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5555             kill 15, $pid;
5556         } else {
5557             die "VM quit/powerdown failed\n";
5558         }
5559     }
5560
5561     # wait again
5562     $timeout = 10;
5563
5564     my $count = 0;
5565     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5566         $count++;
5567         sleep 1;
5568     }
5569
5570     if ($count >= $timeout) {
5571         warn "VM still running - terminating now with SIGKILL\n";
5572         kill 9, $pid;
5573         sleep 1;
5574     }
5575
5576     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5577 }
5578
5579 # Note: use $nocheck to skip tests if VM configuration file exists.
5580 # We need that when migration VMs to other nodes (files already moved)
5581 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5582 sub vm_stop {
5583     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5584
5585     $force = 1 if !defined($force) && !$shutdown;
5586
5587     if ($migratedfrom){
5588         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5589         kill 15, $pid if $pid;
5590         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5591         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5592         return;
5593     }
5594
5595     PVE::QemuConfig->lock_config($vmid, sub {
5596         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5597    });
5598 }
5599
5600 sub vm_reboot {
5601     my ($vmid, $timeout) = @_;
5602
5603     PVE::QemuConfig->lock_config($vmid, sub {
5604         eval {
5605
5606             # only reboot if running, as qmeventd starts it again on a stop event
5607             return if !check_running($vmid);
5608
5609             create_reboot_request($vmid);
5610
5611             my $storecfg = PVE::Storage::config();
5612             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5613
5614         };
5615         if (my $err = $@) {
5616             # avoid that the next normal shutdown will be confused for a reboot
5617             clear_reboot_request($vmid);
5618             die $err;
5619         }
5620    });
5621 }
5622
5623 # note: if using the statestorage parameter, the caller has to check privileges
5624 sub vm_suspend {
5625     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5626
5627     my $conf;
5628     my $path;
5629     my $storecfg;
5630     my $vmstate;
5631
5632     PVE::QemuConfig->lock_config($vmid, sub {
5633
5634         $conf = PVE::QemuConfig->load_config($vmid);
5635
5636         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5637         PVE::QemuConfig->check_lock($conf)
5638             if !($skiplock || $is_backing_up);
5639
5640         die "cannot suspend to disk during backup\n"
5641             if $is_backing_up && $includestate;
5642
5643         if ($includestate) {
5644             $conf->{lock} = 'suspending';
5645             my $date = strftime("%Y-%m-%d", localtime(time()));
5646             $storecfg = PVE::Storage::config();
5647             if (!$statestorage) {
5648                 $statestorage = find_vmstate_storage($conf, $storecfg);
5649                 # check permissions for the storage
5650                 my $rpcenv = PVE::RPCEnvironment::get();
5651                 if ($rpcenv->{type} ne 'cli') {
5652                     my $authuser = $rpcenv->get_user();
5653                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5654                 }
5655             }
5656
5657
5658             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5659                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5660             $path = PVE::Storage::path($storecfg, $vmstate);
5661             PVE::QemuConfig->write_config($vmid, $conf);
5662         } else {
5663             mon_cmd($vmid, "stop");
5664         }
5665     });
5666
5667     if ($includestate) {
5668         # save vm state
5669         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5670
5671         eval {
5672             set_migration_caps($vmid, 1);
5673             mon_cmd($vmid, "savevm-start", statefile => $path);
5674             for(;;) {
5675                 my $state = mon_cmd($vmid, "query-savevm");
5676                 if (!$state->{status}) {
5677                     die "savevm not active\n";
5678                 } elsif ($state->{status} eq 'active') {
5679                     sleep(1);
5680                     next;
5681                 } elsif ($state->{status} eq 'completed') {
5682                     print "State saved, quitting\n";
5683                     last;
5684                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5685                     die "query-savevm failed with error '$state->{error}'\n"
5686                 } else {
5687                     die "query-savevm returned status '$state->{status}'\n";
5688                 }
5689             }
5690         };
5691         my $err = $@;
5692
5693         PVE::QemuConfig->lock_config($vmid, sub {
5694             $conf = PVE::QemuConfig->load_config($vmid);
5695             if ($err) {
5696                 # cleanup, but leave suspending lock, to indicate something went wrong
5697                 eval {
5698                     mon_cmd($vmid, "savevm-end");
5699                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5700                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5701                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5702                     PVE::QemuConfig->write_config($vmid, $conf);
5703                 };
5704                 warn $@ if $@;
5705                 die $err;
5706             }
5707
5708             die "lock changed unexpectedly\n"
5709                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5710
5711             mon_cmd($vmid, "quit");
5712             $conf->{lock} = 'suspended';
5713             PVE::QemuConfig->write_config($vmid, $conf);
5714         });
5715     }
5716 }
5717
5718 sub vm_resume {
5719     my ($vmid, $skiplock, $nocheck) = @_;
5720
5721     PVE::QemuConfig->lock_config($vmid, sub {
5722         my $res = mon_cmd($vmid, 'query-status');
5723         my $resume_cmd = 'cont';
5724         my $reset = 0;
5725
5726         if ($res->{status}) {
5727             return if $res->{status} eq 'running'; # job done, go home
5728             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5729             $reset = 1 if $res->{status} eq 'shutdown';
5730         }
5731
5732         if (!$nocheck) {
5733
5734             my $conf = PVE::QemuConfig->load_config($vmid);
5735
5736             PVE::QemuConfig->check_lock($conf)
5737                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5738         }
5739
5740         if ($reset) {
5741             # required if a VM shuts down during a backup and we get a resume
5742             # request before the backup finishes for example
5743             mon_cmd($vmid, "system_reset");
5744         }
5745         mon_cmd($vmid, $resume_cmd);
5746     });
5747 }
5748
5749 sub vm_sendkey {
5750     my ($vmid, $skiplock, $key) = @_;
5751
5752     PVE::QemuConfig->lock_config($vmid, sub {
5753
5754         my $conf = PVE::QemuConfig->load_config($vmid);
5755
5756         # there is no qmp command, so we use the human monitor command
5757         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5758         die $res if $res ne '';
5759     });
5760 }
5761
5762 # vzdump restore implementaion
5763
5764 sub tar_archive_read_firstfile {
5765     my $archive = shift;
5766
5767     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5768
5769     # try to detect archive type first
5770     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5771         die "unable to open file '$archive'\n";
5772     my $firstfile = <$fh>;
5773     kill 15, $pid;
5774     close $fh;
5775
5776     die "ERROR: archive contaions no data\n" if !$firstfile;
5777     chomp $firstfile;
5778
5779     return $firstfile;
5780 }
5781
5782 sub tar_restore_cleanup {
5783     my ($storecfg, $statfile) = @_;
5784
5785     print STDERR "starting cleanup\n";
5786
5787     if (my $fd = IO::File->new($statfile, "r")) {
5788         while (defined(my $line = <$fd>)) {
5789             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5790                 my $volid = $2;
5791                 eval {
5792                     if ($volid =~ m|^/|) {
5793                         unlink $volid || die 'unlink failed\n';
5794                     } else {
5795                         PVE::Storage::vdisk_free($storecfg, $volid);
5796                     }
5797                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5798                 };
5799                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5800             } else {
5801                 print STDERR "unable to parse line in statfile - $line";
5802             }
5803         }
5804         $fd->close();
5805     }
5806 }
5807
5808 sub restore_file_archive {
5809     my ($archive, $vmid, $user, $opts) = @_;
5810
5811     return restore_vma_archive($archive, $vmid, $user, $opts)
5812         if $archive eq '-';
5813
5814     my $info = PVE::Storage::archive_info($archive);
5815     my $format = $opts->{format} // $info->{format};
5816     my $comp = $info->{compression};
5817
5818     # try to detect archive format
5819     if ($format eq 'tar') {
5820         return restore_tar_archive($archive, $vmid, $user, $opts);
5821     } else {
5822         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5823     }
5824 }
5825
5826 # hepler to remove disks that will not be used after restore
5827 my $restore_cleanup_oldconf = sub {
5828     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5829
5830     PVE::QemuConfig->foreach_volume($oldconf, sub {
5831         my ($ds, $drive) = @_;
5832
5833         return if drive_is_cdrom($drive, 1);
5834
5835         my $volid = $drive->{file};
5836         return if !$volid || $volid =~ m|^/|;
5837
5838         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5839         return if !$path || !$owner || ($owner != $vmid);
5840
5841         # Note: only delete disk we want to restore
5842         # other volumes will become unused
5843         if ($virtdev_hash->{$ds}) {
5844             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5845             if (my $err = $@) {
5846                 warn $err;
5847             }
5848         }
5849     });
5850
5851     # delete vmstate files, after the restore we have no snapshots anymore
5852     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5853         my $snap = $oldconf->{snapshots}->{$snapname};
5854         if ($snap->{vmstate}) {
5855             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5856             if (my $err = $@) {
5857                 warn $err;
5858             }
5859         }
5860     }
5861 };
5862
5863 # Helper to parse vzdump backup device hints
5864 #
5865 # $rpcenv: Environment, used to ckeck storage permissions
5866 # $user: User ID, to check storage permissions
5867 # $storecfg: Storage configuration
5868 # $fh: the file handle for reading the configuration
5869 # $devinfo: should contain device sizes for all backu-up'ed devices
5870 # $options: backup options (pool, default storage)
5871 #
5872 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5873 my $parse_backup_hints = sub {
5874     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5875
5876     my $virtdev_hash = {};
5877
5878     while (defined(my $line = <$fh>)) {
5879         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5880             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5881             die "archive does not contain data for drive '$virtdev'\n"
5882                 if !$devinfo->{$devname};
5883
5884             if (defined($options->{storage})) {
5885                 $storeid = $options->{storage} || 'local';
5886             } elsif (!$storeid) {
5887                 $storeid = 'local';
5888             }
5889             $format = 'raw' if !$format;
5890             $devinfo->{$devname}->{devname} = $devname;
5891             $devinfo->{$devname}->{virtdev} = $virtdev;
5892             $devinfo->{$devname}->{format} = $format;
5893             $devinfo->{$devname}->{storeid} = $storeid;
5894
5895             # check permission on storage
5896             my $pool = $options->{pool}; # todo: do we need that?
5897             if ($user ne 'root@pam') {
5898                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5899             }
5900
5901             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5902         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5903             my $virtdev = $1;
5904             my $drive = parse_drive($virtdev, $2);
5905             if (drive_is_cloudinit($drive)) {
5906                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5907                 $storeid = $options->{storage} if defined ($options->{storage});
5908                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5909                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5910
5911                 $virtdev_hash->{$virtdev} = {
5912                     format => $format,
5913                     storeid => $storeid,
5914                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5915                     is_cloudinit => 1,
5916                 };
5917             }
5918         }
5919     }
5920
5921     return $virtdev_hash;
5922 };
5923
5924 # Helper to allocate and activate all volumes required for a restore
5925 #
5926 # $storecfg: Storage configuration
5927 # $virtdev_hash: as returned by parse_backup_hints()
5928 #
5929 # Returns: { $virtdev => $volid }
5930 my $restore_allocate_devices = sub {
5931     my ($storecfg, $virtdev_hash, $vmid) = @_;
5932
5933     my $map = {};
5934     foreach my $virtdev (sort keys %$virtdev_hash) {
5935         my $d = $virtdev_hash->{$virtdev};
5936         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5937         my $storeid = $d->{storeid};
5938         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5939
5940         # test if requested format is supported
5941         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5942         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5943         $d->{format} = $defFormat if !$supported;
5944
5945         my $name;
5946         if ($d->{is_cloudinit}) {
5947             $name = "vm-$vmid-cloudinit";
5948             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5949             if ($scfg->{path}) {
5950                 $name .= ".$d->{format}";
5951             }
5952         }
5953
5954         my $volid = PVE::Storage::vdisk_alloc(
5955             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5956
5957         print STDERR "new volume ID is '$volid'\n";
5958         $d->{volid} = $volid;
5959
5960         PVE::Storage::activate_volumes($storecfg, [$volid]);
5961
5962         $map->{$virtdev} = $volid;
5963     }
5964
5965     return $map;
5966 };
5967
5968 my $restore_update_config_line = sub {
5969     my ($cookie, $vmid, $map, $line, $unique) = @_;
5970
5971     return '' if $line =~ m/^\#qmdump\#/;
5972     return '' if $line =~ m/^\#vzdump\#/;
5973     return '' if $line =~ m/^lock:/;
5974     return '' if $line =~ m/^unused\d+:/;
5975     return '' if $line =~ m/^parent:/;
5976
5977     my $res = '';
5978
5979     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5980     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5981         # try to convert old 1.X settings
5982         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5983         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5984             my ($model, $macaddr) = split(/\=/, $devconfig);
5985             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5986             my $net = {
5987                 model => $model,
5988                 bridge => "vmbr$ind",
5989                 macaddr => $macaddr,
5990             };
5991             my $netstr = print_net($net);
5992
5993             $res .= "net$cookie->{netcount}: $netstr\n";
5994             $cookie->{netcount}++;
5995         }
5996     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5997         my ($id, $netstr) = ($1, $2);
5998         my $net = parse_net($netstr);
5999         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6000         $netstr = print_net($net);
6001         $res .= "$id: $netstr\n";
6002     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6003         my $virtdev = $1;
6004         my $value = $3;
6005         my $di = parse_drive($virtdev, $value);
6006         if (defined($di->{backup}) && !$di->{backup}) {
6007             $res .= "#$line";
6008         } elsif ($map->{$virtdev}) {
6009             delete $di->{format}; # format can change on restore
6010             $di->{file} = $map->{$virtdev};
6011             $value = print_drive($di);
6012             $res .= "$virtdev: $value\n";
6013         } else {
6014             $res .= $line;
6015         }
6016     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6017         my $vmgenid = $1;
6018         if ($vmgenid ne '0') {
6019             # always generate a new vmgenid if there was a valid one setup
6020             $vmgenid = generate_uuid();
6021         }
6022         $res .= "vmgenid: $vmgenid\n";
6023     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6024         my ($uuid, $uuid_str);
6025         UUID::generate($uuid);
6026         UUID::unparse($uuid, $uuid_str);
6027         my $smbios1 = parse_smbios1($2);
6028         $smbios1->{uuid} = $uuid_str;
6029         $res .= $1.print_smbios1($smbios1)."\n";
6030     } else {
6031         $res .= $line;
6032     }
6033
6034     return $res;
6035 };
6036
6037 my $restore_deactivate_volumes = sub {
6038     my ($storecfg, $devinfo) = @_;
6039
6040     my $vollist = [];
6041     foreach my $devname (keys %$devinfo) {
6042         my $volid = $devinfo->{$devname}->{volid};
6043         push @$vollist, $volid if $volid;
6044     }
6045
6046     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6047 };
6048
6049 my $restore_destroy_volumes = sub {
6050     my ($storecfg, $devinfo) = @_;
6051
6052     foreach my $devname (keys %$devinfo) {
6053         my $volid = $devinfo->{$devname}->{volid};
6054         next if !$volid;
6055         eval {
6056             if ($volid =~ m|^/|) {
6057                 unlink $volid || die 'unlink failed\n';
6058             } else {
6059                 PVE::Storage::vdisk_free($storecfg, $volid);
6060             }
6061             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6062         };
6063         print STDERR "unable to cleanup '$volid' - $@" if $@;
6064     }
6065 };
6066
6067 sub scan_volids {
6068     my ($cfg, $vmid) = @_;
6069
6070     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6071
6072     my $volid_hash = {};
6073     foreach my $storeid (keys %$info) {
6074         foreach my $item (@{$info->{$storeid}}) {
6075             next if !($item->{volid} && $item->{size});
6076             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6077             $volid_hash->{$item->{volid}} = $item;
6078         }
6079     }
6080
6081     return $volid_hash;
6082 }
6083
6084 sub update_disk_config {
6085     my ($vmid, $conf, $volid_hash) = @_;
6086
6087     my $changes;
6088     my $prefix = "VM $vmid";
6089
6090     # used and unused disks
6091     my $referenced = {};
6092
6093     # Note: it is allowed to define multiple storages with same path (alias), so
6094     # we need to check both 'volid' and real 'path' (two different volid can point
6095     # to the same path).
6096
6097     my $referencedpath = {};
6098
6099     # update size info
6100     PVE::QemuConfig->foreach_volume($conf, sub {
6101         my ($opt, $drive) = @_;
6102
6103         my $volid = $drive->{file};
6104         return if !$volid;
6105         my $volume = $volid_hash->{$volid};
6106
6107         # mark volid as "in-use" for next step
6108         $referenced->{$volid} = 1;
6109         if ($volume && (my $path = $volume->{path})) {
6110             $referencedpath->{$path} = 1;
6111         }
6112
6113         return if drive_is_cdrom($drive);
6114         return if !$volume;
6115
6116         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6117         if (defined($updated)) {
6118             $changes = 1;
6119             $conf->{$opt} = print_drive($updated);
6120             print "$prefix ($opt): $msg\n";
6121         }
6122     });
6123
6124     # remove 'unusedX' entry if volume is used
6125     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6126         my ($opt, $drive) = @_;
6127
6128         my $volid = $drive->{file};
6129         return if !$volid;
6130
6131         my $path;
6132         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6133         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6134             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6135             $changes = 1;
6136             delete $conf->{$opt};
6137         }
6138
6139         $referenced->{$volid} = 1;
6140         $referencedpath->{$path} = 1 if $path;
6141     });
6142
6143     foreach my $volid (sort keys %$volid_hash) {
6144         next if $volid =~ m/vm-$vmid-state-/;
6145         next if $referenced->{$volid};
6146         my $path = $volid_hash->{$volid}->{path};
6147         next if !$path; # just to be sure
6148         next if $referencedpath->{$path};
6149         $changes = 1;
6150         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6151         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6152         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6153     }
6154
6155     return $changes;
6156 }
6157
6158 sub rescan {
6159     my ($vmid, $nolock, $dryrun) = @_;
6160
6161     my $cfg = PVE::Storage::config();
6162
6163     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6164     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6165     foreach my $stor (keys %{$cfg->{ids}}) {
6166         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6167     }
6168
6169     print "rescan volumes...\n";
6170     my $volid_hash = scan_volids($cfg, $vmid);
6171
6172     my $updatefn =  sub {
6173         my ($vmid) = @_;
6174
6175         my $conf = PVE::QemuConfig->load_config($vmid);
6176
6177         PVE::QemuConfig->check_lock($conf);
6178
6179         my $vm_volids = {};
6180         foreach my $volid (keys %$volid_hash) {
6181             my $info = $volid_hash->{$volid};
6182             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6183         }
6184
6185         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6186
6187         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6188     };
6189
6190     if (defined($vmid)) {
6191         if ($nolock) {
6192             &$updatefn($vmid);
6193         } else {
6194             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6195         }
6196     } else {
6197         my $vmlist = config_list();
6198         foreach my $vmid (keys %$vmlist) {
6199             if ($nolock) {
6200                 &$updatefn($vmid);
6201             } else {
6202                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6203             }
6204         }
6205     }
6206 }
6207
6208 sub restore_proxmox_backup_archive {
6209     my ($archive, $vmid, $user, $options) = @_;
6210
6211     my $storecfg = PVE::Storage::config();
6212
6213     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6214     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6215
6216     my $fingerprint = $scfg->{fingerprint};
6217     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6218
6219     my $repo = PVE::PBSClient::get_repository($scfg);
6220
6221     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6222     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6223     local $ENV{PBS_PASSWORD} = $password;
6224     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6225
6226     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6227         PVE::Storage::parse_volname($storecfg, $archive);
6228
6229     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6230
6231     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6232
6233     my $tmpdir = "/var/tmp/vzdumptmp$$";
6234     rmtree $tmpdir;
6235     mkpath $tmpdir;
6236
6237     my $conffile = PVE::QemuConfig->config_file($vmid);
6238      # disable interrupts (always do cleanups)
6239     local $SIG{INT} =
6240         local $SIG{TERM} =
6241         local $SIG{QUIT} =
6242         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6243
6244     # Note: $oldconf is undef if VM does not exists
6245     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6246     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6247     my $new_conf_raw = '';
6248
6249     my $rpcenv = PVE::RPCEnvironment::get();
6250     my $devinfo = {};
6251
6252     eval {
6253         # enable interrupts
6254         local $SIG{INT} =
6255             local $SIG{TERM} =
6256             local $SIG{QUIT} =
6257             local $SIG{HUP} =
6258             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6259
6260         my $cfgfn = "$tmpdir/qemu-server.conf";
6261         my $firewall_config_fn = "$tmpdir/fw.conf";
6262         my $index_fn = "$tmpdir/index.json";
6263
6264         my $cmd = "restore";
6265
6266         my $param = [$pbs_backup_name, "index.json", $index_fn];
6267         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6268         my $index = PVE::Tools::file_get_contents($index_fn);
6269         $index = decode_json($index);
6270
6271         # print Dumper($index);
6272         foreach my $info (@{$index->{files}}) {
6273             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6274                 my $devname = $1;
6275                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6276                     $devinfo->{$devname}->{size} = $1;
6277                 } else {
6278                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6279                 }
6280             }
6281         }
6282
6283         my $is_qemu_server_backup = scalar(
6284             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6285         );
6286         if (!$is_qemu_server_backup) {
6287             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6288         }
6289         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6290
6291         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6292         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6293
6294         if ($has_firewall_config) {
6295             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6296             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6297
6298             my $pve_firewall_dir = '/etc/pve/firewall';
6299             mkdir $pve_firewall_dir; # make sure the dir exists
6300             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6301         }
6302
6303         my $fh = IO::File->new($cfgfn, "r") ||
6304             die "unable to read qemu-server.conf - $!\n";
6305
6306         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6307
6308         # fixme: rate limit?
6309
6310         # create empty/temp config
6311         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6312
6313         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6314
6315         # allocate volumes
6316         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6317
6318         if (!$options->{live}) {
6319             foreach my $virtdev (sort keys %$virtdev_hash) {
6320                 my $d = $virtdev_hash->{$virtdev};
6321                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6322
6323                 my $volid = $d->{volid};
6324
6325                 my $path = PVE::Storage::path($storecfg, $volid);
6326
6327                 my $pbs_restore_cmd = [
6328                     '/usr/bin/pbs-restore',
6329                     '--repository', $repo,
6330                     $pbs_backup_name,
6331                     "$d->{devname}.img.fidx",
6332                     $path,
6333                     '--verbose',
6334                     ];
6335
6336                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6337                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6338
6339                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6340                     push @$pbs_restore_cmd, '--skip-zero';
6341                 }
6342
6343                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6344                 print "restore proxmox backup image: $dbg_cmdstring\n";
6345                 run_command($pbs_restore_cmd);
6346             }
6347         }
6348
6349         $fh->seek(0, 0) || die "seek failed - $!\n";
6350
6351         my $cookie = { netcount => 0 };
6352         while (defined(my $line = <$fh>)) {
6353             $new_conf_raw .= $restore_update_config_line->(
6354                 $cookie,
6355                 $vmid,
6356                 $map,
6357                 $line,
6358                 $options->{unique},
6359             );
6360         }
6361
6362         $fh->close();
6363     };
6364     my $err = $@;
6365
6366     if ($err || !$options->{live}) {
6367         $restore_deactivate_volumes->($storecfg, $devinfo);
6368     }
6369
6370     rmtree $tmpdir;
6371
6372     if ($err) {
6373         $restore_destroy_volumes->($storecfg, $devinfo);
6374         die $err;
6375     }
6376
6377     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6378
6379     PVE::Cluster::cfs_update(); # make sure we read new file
6380
6381     eval { rescan($vmid, 1); };
6382     warn $@ if $@;
6383
6384     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6385
6386     if ($options->{live}) {
6387         eval {
6388             # enable interrupts
6389             local $SIG{INT} =
6390                 local $SIG{TERM} =
6391                 local $SIG{QUIT} =
6392                 local $SIG{HUP} =
6393                 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6394
6395             my $conf = PVE::QemuConfig->load_config($vmid);
6396             die "cannot do live-restore for template\n"
6397                 if PVE::QemuConfig->is_template($conf);
6398
6399             pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6400         };
6401
6402         $err = $@;
6403         if ($err) {
6404             warn "Detroying live-restore VM, all temporary data will be lost!\n";
6405             $restore_deactivate_volumes->($storecfg, $devinfo);
6406             $restore_destroy_volumes->($storecfg, $devinfo);
6407             unlink $conffile;
6408             die $err;
6409         }
6410     }
6411 }
6412
6413 sub pbs_live_restore {
6414     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6415
6416     print "Starting VM for live-restore\n";
6417
6418     my $pbs_backing = {};
6419     for my $ds (keys %$restored_disks) {
6420         $ds =~ m/^drive-(.*)$/;
6421         $pbs_backing->{$1} = {
6422             repository => $repo,
6423             snapshot => $snap,
6424             archive => "$ds.img.fidx",
6425         };
6426         $pbs_backing->{$1}->{keyfile} = $keyfile if -e $keyfile;
6427     }
6428
6429     eval {
6430         # make sure HA doesn't interrupt our restore by stopping the VM
6431         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6432             my $cmd = ['ha-manager', 'set',  "vm:$vmid", '--state', 'started'];
6433             PVE::Tools::run_command($cmd);
6434         }
6435
6436         # start VM with backing chain pointing to PBS backup, environment vars
6437         # for PBS driver in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already
6438         # set by our caller
6439         PVE::QemuServer::vm_start_nolock(
6440             $storecfg,
6441             $vmid,
6442             $conf,
6443             {
6444                 paused => 1,
6445                 'pbs-backing' => $pbs_backing,
6446             },
6447             {},
6448         );
6449
6450         my $qmeventd_fd = register_qmeventd_handle($vmid);
6451
6452         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6453         # this will effectively collapse the backing image chain consisting of
6454         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6455         # removes itself once all backing images vanish with 'auto-remove=on')
6456         my $jobs = {};
6457         for my $ds (sort keys %$restored_disks) {
6458             my $job_id = "restore-$ds";
6459             mon_cmd($vmid, 'block-stream',
6460                 'job-id' => $job_id,
6461                 device => "$ds",
6462             );
6463             $jobs->{$job_id} = {};
6464         }
6465
6466         mon_cmd($vmid, 'cont');
6467         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6468
6469         # all jobs finished, remove blockdevs now to disconnect from PBS
6470         for my $ds (sort keys %$restored_disks) {
6471             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6472         }
6473
6474         close($qmeventd_fd);
6475     };
6476
6477     my $err = $@;
6478
6479     if ($err) {
6480         warn "An error occured during live-restore: $err\n";
6481         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6482         die "live-restore failed\n";
6483     }
6484 }
6485
6486 sub restore_vma_archive {
6487     my ($archive, $vmid, $user, $opts, $comp) = @_;
6488
6489     my $readfrom = $archive;
6490
6491     my $cfg = PVE::Storage::config();
6492     my $commands = [];
6493     my $bwlimit = $opts->{bwlimit};
6494
6495     my $dbg_cmdstring = '';
6496     my $add_pipe = sub {
6497         my ($cmd) = @_;
6498         push @$commands, $cmd;
6499         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6500         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6501         $readfrom = '-';
6502     };
6503
6504     my $input = undef;
6505     if ($archive eq '-') {
6506         $input = '<&STDIN';
6507     } else {
6508         # If we use a backup from a PVE defined storage we also consider that
6509         # storage's rate limit:
6510         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6511         if (defined($volid)) {
6512             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6513             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6514             if ($readlimit) {
6515                 print STDERR "applying read rate limit: $readlimit\n";
6516                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6517                 $add_pipe->($cstream);
6518             }
6519         }
6520     }
6521
6522     if ($comp) {
6523         my $info = PVE::Storage::decompressor_info('vma', $comp);
6524         my $cmd = $info->{decompressor};
6525         push @$cmd, $readfrom;
6526         $add_pipe->($cmd);
6527     }
6528
6529     my $tmpdir = "/var/tmp/vzdumptmp$$";
6530     rmtree $tmpdir;
6531
6532     # disable interrupts (always do cleanups)
6533     local $SIG{INT} =
6534         local $SIG{TERM} =
6535         local $SIG{QUIT} =
6536         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6537
6538     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6539     POSIX::mkfifo($mapfifo, 0600);
6540     my $fifofh;
6541     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6542
6543     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6544
6545     my $oldtimeout;
6546     my $timeout = 5;
6547
6548     my $devinfo = {};
6549
6550     my $rpcenv = PVE::RPCEnvironment::get();
6551
6552     my $conffile = PVE::QemuConfig->config_file($vmid);
6553
6554     # Note: $oldconf is undef if VM does not exist
6555     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6556     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6557     my $new_conf_raw = '';
6558
6559     my %storage_limits;
6560
6561     my $print_devmap = sub {
6562         my $cfgfn = "$tmpdir/qemu-server.conf";
6563
6564         # we can read the config - that is already extracted
6565         my $fh = IO::File->new($cfgfn, "r") ||
6566             die "unable to read qemu-server.conf - $!\n";
6567
6568         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6569         if (-f $fwcfgfn) {
6570             my $pve_firewall_dir = '/etc/pve/firewall';
6571             mkdir $pve_firewall_dir; # make sure the dir exists
6572             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6573         }
6574
6575         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6576
6577         foreach my $info (values %{$virtdev_hash}) {
6578             my $storeid = $info->{storeid};
6579             next if defined($storage_limits{$storeid});
6580
6581             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6582             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6583             $storage_limits{$storeid} = $limit * 1024;
6584         }
6585
6586         foreach my $devname (keys %$devinfo) {
6587             die "found no device mapping information for device '$devname'\n"
6588                 if !$devinfo->{$devname}->{virtdev};
6589         }
6590
6591         # create empty/temp config
6592         if ($oldconf) {
6593             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6594             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6595         }
6596
6597         # allocate volumes
6598         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6599
6600         # print restore information to $fifofh
6601         foreach my $virtdev (sort keys %$virtdev_hash) {
6602             my $d = $virtdev_hash->{$virtdev};
6603             next if $d->{is_cloudinit}; # no need to restore cloudinit
6604
6605             my $storeid = $d->{storeid};
6606             my $volid = $d->{volid};
6607
6608             my $map_opts = '';
6609             if (my $limit = $storage_limits{$storeid}) {
6610                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6611             }
6612
6613             my $write_zeros = 1;
6614             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6615                 $write_zeros = 0;
6616             }
6617
6618             my $path = PVE::Storage::path($cfg, $volid);
6619
6620             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6621
6622             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6623         }
6624
6625         $fh->seek(0, 0) || die "seek failed - $!\n";
6626
6627         my $cookie = { netcount => 0 };
6628         while (defined(my $line = <$fh>)) {
6629             $new_conf_raw .= $restore_update_config_line->(
6630                 $cookie,
6631                 $vmid,
6632                 $map,
6633                 $line,
6634                 $opts->{unique},
6635             );
6636         }
6637
6638         $fh->close();
6639     };
6640
6641     eval {
6642         # enable interrupts
6643         local $SIG{INT} =
6644             local $SIG{TERM} =
6645             local $SIG{QUIT} =
6646             local $SIG{HUP} =
6647             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6648         local $SIG{ALRM} = sub { die "got timeout\n"; };
6649
6650         $oldtimeout = alarm($timeout);
6651
6652         my $parser = sub {
6653             my $line = shift;
6654
6655             print "$line\n";
6656
6657             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6658                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6659                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6660             } elsif ($line =~ m/^CTIME: /) {
6661                 # we correctly received the vma config, so we can disable
6662                 # the timeout now for disk allocation (set to 10 minutes, so
6663                 # that we always timeout if something goes wrong)
6664                 alarm(600);
6665                 &$print_devmap();
6666                 print $fifofh "done\n";
6667                 my $tmp = $oldtimeout || 0;
6668                 $oldtimeout = undef;
6669                 alarm($tmp);
6670                 close($fifofh);
6671                 $fifofh = undef;
6672             }
6673         };
6674
6675         print "restore vma archive: $dbg_cmdstring\n";
6676         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6677     };
6678     my $err = $@;
6679
6680     alarm($oldtimeout) if $oldtimeout;
6681
6682     $restore_deactivate_volumes->($cfg, $devinfo);
6683
6684     close($fifofh) if $fifofh;
6685     unlink $mapfifo;
6686     rmtree $tmpdir;
6687
6688     if ($err) {
6689         $restore_destroy_volumes->($cfg, $devinfo);
6690         die $err;
6691     }
6692
6693     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6694
6695     PVE::Cluster::cfs_update(); # make sure we read new file
6696
6697     eval { rescan($vmid, 1); };
6698     warn $@ if $@;
6699
6700     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6701 }
6702
6703 sub restore_tar_archive {
6704     my ($archive, $vmid, $user, $opts) = @_;
6705
6706     if ($archive ne '-') {
6707         my $firstfile = tar_archive_read_firstfile($archive);
6708         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6709             if $firstfile ne 'qemu-server.conf';
6710     }
6711
6712     my $storecfg = PVE::Storage::config();
6713
6714     # avoid zombie disks when restoring over an existing VM -> cleanup first
6715     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6716     # skiplock=1 because qmrestore has set the 'create' lock itself already
6717     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6718     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6719
6720     my $tocmd = "/usr/lib/qemu-server/qmextract";
6721
6722     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6723     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6724     $tocmd .= ' --prealloc' if $opts->{prealloc};
6725     $tocmd .= ' --info' if $opts->{info};
6726
6727     # tar option "xf" does not autodetect compression when read from STDIN,
6728     # so we pipe to zcat
6729     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6730         PVE::Tools::shellquote("--to-command=$tocmd");
6731
6732     my $tmpdir = "/var/tmp/vzdumptmp$$";
6733     mkpath $tmpdir;
6734
6735     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6736     local $ENV{VZDUMP_VMID} = $vmid;
6737     local $ENV{VZDUMP_USER} = $user;
6738
6739     my $conffile = PVE::QemuConfig->config_file($vmid);
6740     my $new_conf_raw = '';
6741
6742     # disable interrupts (always do cleanups)
6743     local $SIG{INT} =
6744         local $SIG{TERM} =
6745         local $SIG{QUIT} =
6746         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6747
6748     eval {
6749         # enable interrupts
6750         local $SIG{INT} =
6751             local $SIG{TERM} =
6752             local $SIG{QUIT} =
6753             local $SIG{HUP} =
6754             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6755
6756         if ($archive eq '-') {
6757             print "extracting archive from STDIN\n";
6758             run_command($cmd, input => "<&STDIN");
6759         } else {
6760             print "extracting archive '$archive'\n";
6761             run_command($cmd);
6762         }
6763
6764         return if $opts->{info};
6765
6766         # read new mapping
6767         my $map = {};
6768         my $statfile = "$tmpdir/qmrestore.stat";
6769         if (my $fd = IO::File->new($statfile, "r")) {
6770             while (defined (my $line = <$fd>)) {
6771                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6772                     $map->{$1} = $2 if $1;
6773                 } else {
6774                     print STDERR "unable to parse line in statfile - $line\n";
6775                 }
6776             }
6777             $fd->close();
6778         }
6779
6780         my $confsrc = "$tmpdir/qemu-server.conf";
6781
6782         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6783
6784         my $cookie = { netcount => 0 };
6785         while (defined (my $line = <$srcfd>)) {
6786             $new_conf_raw .= $restore_update_config_line->(
6787                 $cookie,
6788                 $vmid,
6789                 $map,
6790                 $line,
6791                 $opts->{unique},
6792             );
6793         }
6794
6795         $srcfd->close();
6796     };
6797     if (my $err = $@) {
6798         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6799         die $err;
6800     }
6801
6802     rmtree $tmpdir;
6803
6804     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6805
6806     PVE::Cluster::cfs_update(); # make sure we read new file
6807
6808     eval { rescan($vmid, 1); };
6809     warn $@ if $@;
6810 };
6811
6812 sub foreach_storage_used_by_vm {
6813     my ($conf, $func) = @_;
6814
6815     my $sidhash = {};
6816
6817     PVE::QemuConfig->foreach_volume($conf, sub {
6818         my ($ds, $drive) = @_;
6819         return if drive_is_cdrom($drive);
6820
6821         my $volid = $drive->{file};
6822
6823         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6824         $sidhash->{$sid} = $sid if $sid;
6825     });
6826
6827     foreach my $sid (sort keys %$sidhash) {
6828         &$func($sid);
6829     }
6830 }
6831
6832 my $qemu_snap_storage = {
6833     rbd => 1,
6834 };
6835 sub do_snapshots_with_qemu {
6836     my ($storecfg, $volid) = @_;
6837
6838     my $storage_name = PVE::Storage::parse_volume_id($volid);
6839     my $scfg = $storecfg->{ids}->{$storage_name};
6840     die "could not find storage '$storage_name'\n" if !defined($scfg);
6841
6842     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6843         return 1;
6844     }
6845
6846     if ($volid =~ m/\.(qcow2|qed)$/){
6847         return 1;
6848     }
6849
6850     return;
6851 }
6852
6853 sub qga_check_running {
6854     my ($vmid, $nowarn) = @_;
6855
6856     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6857     if ($@) {
6858         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6859         return 0;
6860     }
6861     return 1;
6862 }
6863
6864 sub template_create {
6865     my ($vmid, $conf, $disk) = @_;
6866
6867     my $storecfg = PVE::Storage::config();
6868
6869     PVE::QemuConfig->foreach_volume($conf, sub {
6870         my ($ds, $drive) = @_;
6871
6872         return if drive_is_cdrom($drive);
6873         return if $disk && $ds ne $disk;
6874
6875         my $volid = $drive->{file};
6876         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6877
6878         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6879         $drive->{file} = $voliddst;
6880         $conf->{$ds} = print_drive($drive);
6881         PVE::QemuConfig->write_config($vmid, $conf);
6882     });
6883 }
6884
6885 sub convert_iscsi_path {
6886     my ($path) = @_;
6887
6888     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6889         my $portal = $1;
6890         my $target = $2;
6891         my $lun = $3;
6892
6893         my $initiator_name = get_initiator_name();
6894
6895         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6896                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6897     }
6898
6899     die "cannot convert iscsi path '$path', unkown format\n";
6900 }
6901
6902 sub qemu_img_convert {
6903     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6904
6905     my $storecfg = PVE::Storage::config();
6906     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6907     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6908
6909     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6910
6911     my $cachemode;
6912     my $src_path;
6913     my $src_is_iscsi = 0;
6914     my $src_format;
6915
6916     if ($src_storeid) {
6917         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6918         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6919         $src_format = qemu_img_format($src_scfg, $src_volname);
6920         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6921         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6922         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6923     } elsif (-f $src_volid) {
6924         $src_path = $src_volid;
6925         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6926             $src_format = $1;
6927         }
6928     }
6929
6930     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6931
6932     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6933     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6934     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6935     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6936
6937     my $cmd = [];
6938     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6939     push @$cmd, '-l', "snapshot.name=$snapname"
6940         if $snapname && $src_format && $src_format eq "qcow2";
6941     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6942     push @$cmd, '-T', $cachemode if defined($cachemode);
6943
6944     if ($src_is_iscsi) {
6945         push @$cmd, '--image-opts';
6946         $src_path = convert_iscsi_path($src_path);
6947     } elsif ($src_format) {
6948         push @$cmd, '-f', $src_format;
6949     }
6950
6951     if ($dst_is_iscsi) {
6952         push @$cmd, '--target-image-opts';
6953         $dst_path = convert_iscsi_path($dst_path);
6954     } else {
6955         push @$cmd, '-O', $dst_format;
6956     }
6957
6958     push @$cmd, $src_path;
6959
6960     if (!$dst_is_iscsi && $is_zero_initialized) {
6961         push @$cmd, "zeroinit:$dst_path";
6962     } else {
6963         push @$cmd, $dst_path;
6964     }
6965
6966     my $parser = sub {
6967         my $line = shift;
6968         if($line =~ m/\((\S+)\/100\%\)/){
6969             my $percent = $1;
6970             my $transferred = int($size * $percent / 100);
6971             my $remaining = $size - $transferred;
6972
6973             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6974         }
6975
6976     };
6977
6978     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6979     my $err = $@;
6980     die "copy failed: $err" if $err;
6981 }
6982
6983 sub qemu_img_format {
6984     my ($scfg, $volname) = @_;
6985
6986     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6987         return $1;
6988     } else {
6989         return "raw";
6990     }
6991 }
6992
6993 sub qemu_drive_mirror {
6994     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6995
6996     $jobs = {} if !$jobs;
6997
6998     my $qemu_target;
6999     my $format;
7000     $jobs->{"drive-$drive"} = {};
7001
7002     if ($dst_volid =~ /^nbd:/) {
7003         $qemu_target = $dst_volid;
7004         $format = "nbd";
7005     } else {
7006         my $storecfg = PVE::Storage::config();
7007         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7008
7009         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7010
7011         $format = qemu_img_format($dst_scfg, $dst_volname);
7012
7013         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7014
7015         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7016     }
7017
7018     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7019     $opts->{format} = $format if $format;
7020
7021     if (defined($src_bitmap)) {
7022         $opts->{sync} = 'incremental';
7023         $opts->{bitmap} = $src_bitmap;
7024         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7025     }
7026
7027     if (defined($bwlimit)) {
7028         $opts->{speed} = $bwlimit * 1024;
7029         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7030     } else {
7031         print "drive mirror is starting for drive-$drive\n";
7032     }
7033
7034     # if a job already runs for this device we get an error, catch it for cleanup
7035     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7036     if (my $err = $@) {
7037         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7038         warn "$@\n" if $@;
7039         die "mirroring error: $err\n";
7040     }
7041
7042     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7043 }
7044
7045 # $completion can be either
7046 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7047 # 'cancel': wait until all jobs are ready, block-job-cancel them
7048 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7049 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7050 sub qemu_drive_mirror_monitor {
7051     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7052
7053     $completion //= 'complete';
7054     $op //= "mirror";
7055
7056     eval {
7057         my $err_complete = 0;
7058
7059         while (1) {
7060             die "block job ('$op') timed out\n" if $err_complete > 300;
7061
7062             my $stats = mon_cmd($vmid, "query-block-jobs");
7063
7064             my $running_jobs = {};
7065             foreach my $stat (@$stats) {
7066                 next if $stat->{type} ne $op;
7067                 $running_jobs->{$stat->{device}} = $stat;
7068             }
7069
7070             my $readycounter = 0;
7071
7072             foreach my $job (keys %$jobs) {
7073
7074                 my $vanished = !defined($running_jobs->{$job});
7075                 my $complete = defined($jobs->{$job}->{complete}) && $vanished;
7076                 if($complete || ($vanished && $completion eq 'auto')) {
7077                     print "$job: finished\n";
7078                     delete $jobs->{$job};
7079                     next;
7080                 }
7081
7082                 die "$job: '$op' has been cancelled\n" if !defined($running_jobs->{$job});
7083
7084                 my $busy = $running_jobs->{$job}->{busy};
7085                 my $ready = $running_jobs->{$job}->{ready};
7086                 if (my $total = $running_jobs->{$job}->{len}) {
7087                     my $transferred = $running_jobs->{$job}->{offset} || 0;
7088                     my $remaining = $total - $transferred;
7089                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7090
7091                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7092                 }
7093
7094                 $readycounter++ if $running_jobs->{$job}->{ready};
7095             }
7096
7097             last if scalar(keys %$jobs) == 0;
7098
7099             if ($readycounter == scalar(keys %$jobs)) {
7100                 print "all '$op' jobs are ready\n";
7101
7102                 # do the complete later (or has already been done)
7103                 last if $completion eq 'skip' || $completion eq 'auto';
7104
7105                 if ($vmiddst && $vmiddst != $vmid) {
7106                     my $agent_running = $qga && qga_check_running($vmid);
7107                     if ($agent_running) {
7108                         print "freeze filesystem\n";
7109                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7110                     } else {
7111                         print "suspend vm\n";
7112                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7113                     }
7114
7115                     # if we clone a disk for a new target vm, we don't switch the disk
7116                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7117
7118                     if ($agent_running) {
7119                         print "unfreeze filesystem\n";
7120                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7121                     } else {
7122                         print "resume vm\n";
7123                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7124                     }
7125
7126                     last;
7127                 } else {
7128
7129                     foreach my $job (keys %$jobs) {
7130                         # try to switch the disk if source and destination are on the same guest
7131                         print "$job: Completing block job...\n";
7132
7133                         my $op;
7134                         if ($completion eq 'complete') {
7135                             $op = 'block-job-complete';
7136                         } elsif ($completion eq 'cancel') {
7137                             $op = 'block-job-cancel';
7138                         } else {
7139                             die "invalid completion value: $completion\n";
7140                         }
7141                         eval { mon_cmd($vmid, $op, device => $job) };
7142                         if ($@ =~ m/cannot be completed/) {
7143                             print "$job: Block job cannot be completed, try again.\n";
7144                             $err_complete++;
7145                         }else {
7146                             print "$job: Completed successfully.\n";
7147                             $jobs->{$job}->{complete} = 1;
7148                         }
7149                     }
7150                 }
7151             }
7152             sleep 1;
7153         }
7154     };
7155     my $err = $@;
7156
7157     if ($err) {
7158         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7159         die "block job ('$op') error: $err";
7160     }
7161
7162 }
7163
7164 sub qemu_blockjobs_cancel {
7165     my ($vmid, $jobs) = @_;
7166
7167     foreach my $job (keys %$jobs) {
7168         print "$job: Cancelling block job\n";
7169         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7170         $jobs->{$job}->{cancel} = 1;
7171     }
7172
7173     while (1) {
7174         my $stats = mon_cmd($vmid, "query-block-jobs");
7175
7176         my $running_jobs = {};
7177         foreach my $stat (@$stats) {
7178             $running_jobs->{$stat->{device}} = $stat;
7179         }
7180
7181         foreach my $job (keys %$jobs) {
7182
7183             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7184                 print "$job: Done.\n";
7185                 delete $jobs->{$job};
7186             }
7187         }
7188
7189         last if scalar(keys %$jobs) == 0;
7190
7191         sleep 1;
7192     }
7193 }
7194
7195 sub clone_disk {
7196     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7197         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7198
7199     my $newvolid;
7200
7201     if (!$full) {
7202         print "create linked clone of drive $drivename ($drive->{file})\n";
7203         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7204         push @$newvollist, $newvolid;
7205     } else {
7206
7207         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7208         $storeid = $storage if $storage;
7209
7210         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7211
7212         print "create full clone of drive $drivename ($drive->{file})\n";
7213         my $name = undef;
7214         my $size = undef;
7215         if (drive_is_cloudinit($drive)) {
7216             $name = "vm-$newvmid-cloudinit";
7217             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7218             if ($scfg->{path}) {
7219                 $name .= ".$dst_format";
7220             }
7221             $snapname = undef;
7222             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7223         } elsif ($drivename eq 'efidisk0') {
7224             $size = get_efivars_size($conf);
7225         } else {
7226             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7227         }
7228         $newvolid = PVE::Storage::vdisk_alloc(
7229             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7230         );
7231         push @$newvollist, $newvolid;
7232
7233         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7234
7235         if (drive_is_cloudinit($drive)) {
7236             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7237             # if this is the case, we have to complete any block-jobs still there from
7238             # previous drive-mirrors
7239             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7240                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7241             }
7242             goto no_data_clone;
7243         }
7244
7245         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7246         if (!$running || $snapname) {
7247             # TODO: handle bwlimits
7248             if ($drivename eq 'efidisk0') {
7249                 # the relevant data on the efidisk may be smaller than the source
7250                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7251                 # that is given by the OVMF_VARS.fd
7252                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7253                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7254
7255                 # better for Ceph if block size is not too small, see bug #3324
7256                 my $bs = 1024*1024;
7257
7258                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7259                     "if=$src_path", "of=$dst_path"]);
7260             } else {
7261                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7262             }
7263         } else {
7264
7265             my $kvmver = get_running_qemu_version ($vmid);
7266             if (!min_version($kvmver, 2, 7)) {
7267                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7268                     if $drive->{iothread};
7269             }
7270
7271             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7272                 $completion, $qga, $bwlimit);
7273         }
7274     }
7275
7276 no_data_clone:
7277     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7278
7279     my $disk = $drive;
7280     $disk->{format} = undef;
7281     $disk->{file} = $newvolid;
7282     $disk->{size} = $size if defined($size);
7283
7284     return $disk;
7285 }
7286
7287 sub get_running_qemu_version {
7288     my ($vmid) = @_;
7289     my $res = mon_cmd($vmid, "query-version");
7290     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7291 }
7292
7293 sub qemu_use_old_bios_files {
7294     my ($machine_type) = @_;
7295
7296     return if !$machine_type;
7297
7298     my $use_old_bios_files = undef;
7299
7300     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7301         $machine_type = $1;
7302         $use_old_bios_files = 1;
7303     } else {
7304         my $version = extract_version($machine_type, kvm_user_version());
7305         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7306         # load new efi bios files on migration. So this hack is required to allow
7307         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7308         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7309         $use_old_bios_files = !min_version($version, 2, 4);
7310     }
7311
7312     return ($use_old_bios_files, $machine_type);
7313 }
7314
7315 sub get_efivars_size {
7316     my ($conf) = @_;
7317     my $arch = get_vm_arch($conf);
7318     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7319     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7320     return -s $ovmf_vars;
7321 }
7322
7323 sub update_efidisk_size {
7324     my ($conf) = @_;
7325
7326     return if !defined($conf->{efidisk0});
7327
7328     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7329     $disk->{size} = get_efivars_size($conf);
7330     $conf->{efidisk0} = print_drive($disk);
7331
7332     return;
7333 }
7334
7335 sub create_efidisk($$$$$) {
7336     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7337
7338     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7339     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7340
7341     my $vars_size_b = -s $ovmf_vars;
7342     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7343     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7344     PVE::Storage::activate_volumes($storecfg, [$volid]);
7345
7346     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7347     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7348
7349     return ($volid, $size/1024);
7350 }
7351
7352 sub vm_iothreads_list {
7353     my ($vmid) = @_;
7354
7355     my $res = mon_cmd($vmid, 'query-iothreads');
7356
7357     my $iothreads = {};
7358     foreach my $iothread (@$res) {
7359         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7360     }
7361
7362     return $iothreads;
7363 }
7364
7365 sub scsihw_infos {
7366     my ($conf, $drive) = @_;
7367
7368     my $maxdev = 0;
7369
7370     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7371         $maxdev = 7;
7372     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7373         $maxdev = 1;
7374     } else {
7375         $maxdev = 256;
7376     }
7377
7378     my $controller = int($drive->{index} / $maxdev);
7379     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7380         ? "virtioscsi"
7381         : "scsihw";
7382
7383     return ($maxdev, $controller, $controller_prefix);
7384 }
7385
7386 sub windows_version {
7387     my ($ostype) = @_;
7388
7389     return 0 if !$ostype;
7390
7391     my $winversion = 0;
7392
7393     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7394         $winversion = 5;
7395     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7396         $winversion = 6;
7397     } elsif ($ostype =~ m/^win(\d+)$/) {
7398         $winversion = $1;
7399     }
7400
7401     return $winversion;
7402 }
7403
7404 sub resolve_dst_disk_format {
7405         my ($storecfg, $storeid, $src_volname, $format) = @_;
7406         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7407
7408         if (!$format) {
7409             # if no target format is specified, use the source disk format as hint
7410             if ($src_volname) {
7411                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7412                 $format = qemu_img_format($scfg, $src_volname);
7413             } else {
7414                 return $defFormat;
7415             }
7416         }
7417
7418         # test if requested format is supported - else use default
7419         my $supported = grep { $_ eq $format } @$validFormats;
7420         $format = $defFormat if !$supported;
7421         return $format;
7422 }
7423
7424 # NOTE: if this logic changes, please update docs & possibly gui logic
7425 sub find_vmstate_storage {
7426     my ($conf, $storecfg) = @_;
7427
7428     # first, return storage from conf if set
7429     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7430
7431     my ($target, $shared, $local);
7432
7433     foreach_storage_used_by_vm($conf, sub {
7434         my ($sid) = @_;
7435         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7436         my $dst = $scfg->{shared} ? \$shared : \$local;
7437         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7438     });
7439
7440     # second, use shared storage where VM has at least one disk
7441     # third, use local storage where VM has at least one disk
7442     # fall back to local storage
7443     $target = $shared // $local // 'local';
7444
7445     return $target;
7446 }
7447
7448 sub generate_uuid {
7449     my ($uuid, $uuid_str);
7450     UUID::generate($uuid);
7451     UUID::unparse($uuid, $uuid_str);
7452     return $uuid_str;
7453 }
7454
7455 sub generate_smbios1_uuid {
7456     return "uuid=".generate_uuid();
7457 }
7458
7459 sub nbd_stop {
7460     my ($vmid) = @_;
7461
7462     mon_cmd($vmid, 'nbd-server-stop');
7463 }
7464
7465 sub create_reboot_request {
7466     my ($vmid) = @_;
7467     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7468         or die "failed to create reboot trigger file: $!\n";
7469     close($fh);
7470 }
7471
7472 sub clear_reboot_request {
7473     my ($vmid) = @_;
7474     my $path = "/run/qemu-server/$vmid.reboot";
7475     my $res = 0;
7476
7477     $res = unlink($path);
7478     die "could not remove reboot request for $vmid: $!"
7479         if !$res && $! != POSIX::ENOENT;
7480
7481     return $res;
7482 }
7483
7484 sub bootorder_from_legacy {
7485     my ($conf, $bootcfg) = @_;
7486
7487     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7488     my $bootindex_hash = {};
7489     my $i = 1;
7490     foreach my $o (split(//, $boot)) {
7491         $bootindex_hash->{$o} = $i*100;
7492         $i++;
7493     }
7494
7495     my $bootorder = {};
7496
7497     PVE::QemuConfig->foreach_volume($conf, sub {
7498         my ($ds, $drive) = @_;
7499
7500         if (drive_is_cdrom ($drive, 1)) {
7501             if ($bootindex_hash->{d}) {
7502                 $bootorder->{$ds} = $bootindex_hash->{d};
7503                 $bootindex_hash->{d} += 1;
7504             }
7505         } elsif ($bootindex_hash->{c}) {
7506             $bootorder->{$ds} = $bootindex_hash->{c}
7507                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7508             $bootindex_hash->{c} += 1;
7509         }
7510     });
7511
7512     if ($bootindex_hash->{n}) {
7513         for (my $i = 0; $i < $MAX_NETS; $i++) {
7514             my $netname = "net$i";
7515             next if !$conf->{$netname};
7516             $bootorder->{$netname} = $bootindex_hash->{n};
7517             $bootindex_hash->{n} += 1;
7518         }
7519     }
7520
7521     return $bootorder;
7522 }
7523
7524 # Generate default device list for 'boot: order=' property. Matches legacy
7525 # default boot order, but with explicit device names. This is important, since
7526 # the fallback for when neither 'order' nor the old format is specified relies
7527 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7528 sub get_default_bootdevices {
7529     my ($conf) = @_;
7530
7531     my @ret = ();
7532
7533     # harddisk
7534     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7535     push @ret, $first if $first;
7536
7537     # cdrom
7538     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7539     push @ret, $first if $first;
7540
7541     # network
7542     for (my $i = 0; $i < $MAX_NETS; $i++) {
7543         my $netname = "net$i";
7544         next if !$conf->{$netname};
7545         push @ret, $netname;
7546         last;
7547     }
7548
7549     return \@ret;
7550 }
7551
7552 sub device_bootorder {
7553     my ($conf) = @_;
7554
7555     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7556
7557     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7558
7559     my $bootorder = {};
7560     if (!defined($boot) || $boot->{legacy}) {
7561         $bootorder = bootorder_from_legacy($conf, $boot);
7562     } elsif ($boot->{order}) {
7563         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7564         for my $dev (PVE::Tools::split_list($boot->{order})) {
7565             $bootorder->{$dev} = $i++;
7566         }
7567     }
7568
7569     return $bootorder;
7570 }
7571
7572 sub register_qmeventd_handle {
7573     my ($vmid) = @_;
7574
7575     my $fh;
7576     my $peer = "/var/run/qmeventd.sock";
7577     my $count = 0;
7578
7579     for (;;) {
7580         $count++;
7581         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7582         last if $fh;
7583         if ($! != EINTR && $! != EAGAIN) {
7584             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7585         }
7586         if ($count > 4) {
7587             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7588               . "after $count retries\n";
7589         }
7590         usleep(25000);
7591     }
7592
7593     # send handshake to mark VM as backing up
7594     print $fh to_json({vzdump => {vmid => "$vmid"}});
7595
7596     # return handle to be closed later when inhibit is no longer required
7597     return $fh;
7598 }
7599
7600 # bash completion helper
7601
7602 sub complete_backup_archives {
7603     my ($cmdname, $pname, $cvalue) = @_;
7604
7605     my $cfg = PVE::Storage::config();
7606
7607     my $storeid;
7608
7609     if ($cvalue =~ m/^([^:]+):/) {
7610         $storeid = $1;
7611     }
7612
7613     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7614
7615     my $res = [];
7616     foreach my $id (keys %$data) {
7617         foreach my $item (@{$data->{$id}}) {
7618             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7619             push @$res, $item->{volid} if defined($item->{volid});
7620         }
7621     }
7622
7623     return $res;
7624 }
7625
7626 my $complete_vmid_full = sub {
7627     my ($running) = @_;
7628
7629     my $idlist = vmstatus();
7630
7631     my $res = [];
7632
7633     foreach my $id (keys %$idlist) {
7634         my $d = $idlist->{$id};
7635         if (defined($running)) {
7636             next if $d->{template};
7637             next if $running && $d->{status} ne 'running';
7638             next if !$running && $d->{status} eq 'running';
7639         }
7640         push @$res, $id;
7641
7642     }
7643     return $res;
7644 };
7645
7646 sub complete_vmid {
7647     return &$complete_vmid_full();
7648 }
7649
7650 sub complete_vmid_stopped {
7651     return &$complete_vmid_full(0);
7652 }
7653
7654 sub complete_vmid_running {
7655     return &$complete_vmid_full(1);
7656 }
7657
7658 sub complete_storage {
7659
7660     my $cfg = PVE::Storage::config();
7661     my $ids = $cfg->{ids};
7662
7663     my $res = [];
7664     foreach my $sid (keys %$ids) {
7665         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7666         next if !$ids->{$sid}->{content}->{images};
7667         push @$res, $sid;
7668     }
7669
7670     return $res;
7671 }
7672
7673 sub complete_migration_storage {
7674     my ($cmd, $param, $current_value, $all_args) = @_;
7675
7676     my $targetnode = @$all_args[1];
7677
7678     my $cfg = PVE::Storage::config();
7679     my $ids = $cfg->{ids};
7680
7681     my $res = [];
7682     foreach my $sid (keys %$ids) {
7683         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7684         next if !$ids->{$sid}->{content}->{images};
7685         push @$res, $sid;
7686     }
7687
7688     return $res;
7689 }
7690
7691 sub vm_is_paused {
7692     my ($vmid) = @_;
7693     my $qmpstatus = eval {
7694         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7695         mon_cmd($vmid, "query-status");
7696     };
7697     warn "$@\n" if $@;
7698     return $qmpstatus && $qmpstatus->{status} eq "paused";
7699 }
7700
7701 1;