PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after cloning/moving a disk.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 968
 969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 970 dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $usb_fmt = {
1002     host => {
1003         default_key => 1,
1004         type => 'string', format => 'pve-qm-usb-device',
1005         format_description => 'HOSTUSBDEVICE|spice',
1006         description => <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1008
1009  'bus-port(.port)*' (decimal numbers) or
1010  'vendor_id:product_id' (hexadeciaml numbers) or
1011  'spice'
1012
1013 You can use the 'lsusb -t' command to list existing usb devices.
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1017
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1019 EODESCR
1020     },
1021     usb3 => {
1022         optional => 1,
1023         type => 'boolean',
1024         description => "Specifies whether if given host option is a USB3 device or port.",
1025         default => 0,
1026     },
1027 };
1028
1029 my $usbdesc = {
1030     optional => 1,
1031     type => 'string', format => $usb_fmt,
1032     description => "Configure an USB device (n is 0 to 4).",
1033 };
1034 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1035
1036 my $serialdesc = {
1037         optional => 1,
1038         type => 'string',
1039         pattern => '(/dev/.+|socket)',
1040         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1041         verbose_description =>  <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1045
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1048
1049 CAUTION: Experimental! User reported problems with this option.
1050 EODESCR
1051 };
1052
1053 my $paralleldesc= {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1057         description =>  "Map host parallel devices (n is 0 to 2).",
1058         verbose_description =>  <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1060
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1069     $confdesc->{"parallel$i"} = $paralleldesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1073     $confdesc->{"serial$i"} = $serialdesc;
1074 }
1075
1076 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1077     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1078 }
1079
1080 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1081     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1082 }
1083
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1085     $confdesc->{"usb$i"} = $usbdesc;
1086 }
1087
1088 my $boot_fmt = {
1089     legacy => {
1090         optional => 1,
1091         default_key => 1,
1092         type => 'string',
1093         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094                      . " Deprecated, use 'order=' instead.",
1095         pattern => '[acdn]{1,4}',
1096         format_description => "[acdn]{1,4}",
1097
1098         # note: this is also the fallback if boot: is not given at all
1099         default => 'cdn',
1100     },
1101     order => {
1102         optional => 1,
1103         type => 'string',
1104         format => 'pve-qm-bootdev-list',
1105         format_description => "device[;device...]",
1106         description => <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1108
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1112
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1116
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1118 EODESC
1119     },
1120 };
1121 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1122
1123 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1124 sub verify_bootdev {
1125     my ($dev, $noerr) = @_;
1126
1127     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1128
1129     my $check = sub {
1130         my ($base) = @_;
1131         return 0 if $dev !~ m/^$base\d+$/;
1132         return 0 if !$confdesc->{$dev};
1133         return 1;
1134     };
1135
1136     return $dev if $check->("net");
1137     return $dev if $check->("usb");
1138     return $dev if $check->("hostpci");
1139
1140     return if $noerr;
1141     die "invalid boot device '$dev'\n";
1142 }
1143
1144 sub print_bootorder {
1145     my ($devs) = @_;
1146     return "" if !@$devs;
1147     my $data = { order => join(';', @$devs) };
1148     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1149 }
1150
1151 my $kvm_api_version = 0;
1152
1153 sub kvm_version {
1154     return $kvm_api_version if $kvm_api_version;
1155
1156     open my $fh, '<', '/dev/kvm' or return;
1157
1158     # 0xae00 => KVM_GET_API_VERSION
1159     $kvm_api_version = ioctl($fh, 0xae00, 0);
1160     close($fh);
1161
1162     return $kvm_api_version;
1163 }
1164
1165 my $kvm_user_version = {};
1166 my $kvm_mtime = {};
1167
1168 sub kvm_user_version {
1169     my ($binary) = @_;
1170
1171     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1172     my $st = stat($binary);
1173
1174     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176         $cachedmtime == $st->mtime;
1177
1178     $kvm_user_version->{$binary} = 'unknown';
1179     $kvm_mtime->{$binary} = $st->mtime;
1180
1181     my $code = sub {
1182         my $line = shift;
1183         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184             $kvm_user_version->{$binary} = $2;
1185         }
1186     };
1187
1188     eval { run_command([$binary, '--version'], outfunc => $code); };
1189     warn $@ if $@;
1190
1191     return $kvm_user_version->{$binary};
1192
1193 }
1194 my sub extract_version {
1195     my ($machine_type, $version) = @_;
1196     $version = kvm_user_version() if !defined($version);
1197     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1198 }
1199
1200 sub kernel_has_vhost_net {
1201     return -c '/dev/vhost-net';
1202 }
1203
1204 sub option_exists {
1205     my $key = shift;
1206     return defined($confdesc->{$key});
1207 }
1208
1209 my $cdrom_path;
1210 sub get_cdrom_path {
1211
1212     return  $cdrom_path if $cdrom_path;
1213
1214     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1215     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1216     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1217 }
1218
1219 sub get_iso_path {
1220     my ($storecfg, $vmid, $cdrom) = @_;
1221
1222     if ($cdrom eq 'cdrom') {
1223         return get_cdrom_path();
1224     } elsif ($cdrom eq 'none') {
1225         return '';
1226     } elsif ($cdrom =~ m|^/|) {
1227         return $cdrom;
1228     } else {
1229         return PVE::Storage::path($storecfg, $cdrom);
1230     }
1231 }
1232
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id {
1235     my ($vmid, $file, $media) = @_;
1236
1237      if (!($file eq 'none' || $file eq 'cdrom' ||
1238           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1239
1240         return if $file =~ m|/|;
1241
1242         if ($media && $media eq 'cdrom') {
1243             $file = "local:iso/$file";
1244         } else {
1245             $file = "local:$vmid/$file";
1246         }
1247     }
1248
1249     return $file;
1250 }
1251
1252 sub verify_media_type {
1253     my ($opt, $vtype, $media) = @_;
1254
1255     return if !$media;
1256
1257     my $etype;
1258     if ($media eq 'disk') {
1259         $etype = 'images';
1260     } elsif ($media eq 'cdrom') {
1261         $etype = 'iso';
1262     } else {
1263         die "internal error";
1264     }
1265
1266     return if ($vtype eq $etype);
1267
1268     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1269 }
1270
1271 sub cleanup_drive_path {
1272     my ($opt, $storecfg, $drive) = @_;
1273
1274     # try to convert filesystem paths to volume IDs
1275
1276     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1277         ($drive->{file} !~ m|^/dev/.+|) &&
1278         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1279         ($drive->{file} !~ m/^\d+$/)) {
1280         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1281         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1282             if !$vtype;
1283         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1284         verify_media_type($opt, $vtype, $drive->{media});
1285         $drive->{file} = $volid;
1286     }
1287
1288     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1289 }
1290
1291 sub parse_hotplug_features {
1292     my ($data) = @_;
1293
1294     my $res = {};
1295
1296     return $res if $data eq '0';
1297
1298     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1299
1300     foreach my $feature (PVE::Tools::split_list($data)) {
1301         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1302             $res->{$1} = 1;
1303         } else {
1304             die "invalid hotplug feature '$feature'\n";
1305         }
1306     }
1307     return $res;
1308 }
1309
1310 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1311 sub pve_verify_hotplug_features {
1312     my ($value, $noerr) = @_;
1313
1314     return $value if parse_hotplug_features($value);
1315
1316     return if $noerr;
1317
1318     die "unable to parse hotplug option\n";
1319 }
1320
1321 sub scsi_inquiry {
1322     my($fh, $noerr) = @_;
1323
1324     my $SG_IO = 0x2285;
1325     my $SG_GET_VERSION_NUM = 0x2282;
1326
1327     my $versionbuf = "\x00" x 8;
1328     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1329     if (!$ret) {
1330         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1331         return;
1332     }
1333     my $version = unpack("I", $versionbuf);
1334     if ($version < 30000) {
1335         die "scsi generic interface too old\n"  if !$noerr;
1336         return;
1337     }
1338
1339     my $buf = "\x00" x 36;
1340     my $sensebuf = "\x00" x 8;
1341     my $cmd = pack("C x3 C x1", 0x12, 36);
1342
1343     # see /usr/include/scsi/sg.h
1344     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1345
1346     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347                       length($sensebuf), 0, length($buf), $buf,
1348                       $cmd, $sensebuf, 6000);
1349
1350     $ret = ioctl($fh, $SG_IO, $packet);
1351     if (!$ret) {
1352         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1353         return;
1354     }
1355
1356     my @res = unpack($sg_io_hdr_t, $packet);
1357     if ($res[17] || $res[18]) {
1358         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1359         return;
1360     }
1361
1362     my $res = {};
1363     (my $byte0, my $byte1, $res->{vendor},
1364      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1365
1366     $res->{removable} = $byte1 & 128 ? 1 : 0;
1367     $res->{type} = $byte0 & 31;
1368
1369     return $res;
1370 }
1371
1372 sub path_is_scsi {
1373     my ($path) = @_;
1374
1375     my $fh = IO::File->new("+<$path") || return;
1376     my $res = scsi_inquiry($fh, 1);
1377     close($fh);
1378
1379     return $res;
1380 }
1381
1382 sub print_tabletdevice_full {
1383     my ($conf, $arch) = @_;
1384
1385     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1386
1387     # we use uhci for old VMs because tablet driver was buggy in older qemu
1388     my $usbbus;
1389     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1390         $usbbus = 'ehci';
1391     } else {
1392         $usbbus = 'uhci';
1393     }
1394
1395     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1396 }
1397
1398 sub print_keyboarddevice_full {
1399     my ($conf, $arch, $machine) = @_;
1400
1401     return if $arch ne 'aarch64';
1402
1403     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1404 }
1405
1406 sub print_drivedevice_full {
1407     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1408
1409     my $device = '';
1410     my $maxdev = 0;
1411
1412     my $drive_id = "$drive->{interface}$drive->{index}";
1413     if ($drive->{interface} eq 'virtio') {
1414         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1415         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1417     } elsif ($drive->{interface} eq 'scsi') {
1418
1419         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1420         my $unit = $drive->{index} % $maxdev;
1421         my $devicetype = 'hd';
1422         my $path = '';
1423         if (drive_is_cdrom($drive)) {
1424             $devicetype = 'cd';
1425         } else {
1426             if ($drive->{file} =~ m|^/|) {
1427                 $path = $drive->{file};
1428                 if (my $info = path_is_scsi($path)) {
1429                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1430                         $devicetype = 'block';
1431                     } elsif ($info->{type} == 1) { # tape
1432                         $devicetype = 'generic';
1433                     }
1434                 }
1435             } else {
1436                  $path = PVE::Storage::path($storecfg, $drive->{file});
1437             }
1438
1439             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440             my $version = extract_version($machine_type, kvm_user_version());
1441             if ($path =~ m/^iscsi\:\/\// &&
1442                !min_version($version, 4, 1)) {
1443                 $devicetype = 'generic';
1444             }
1445         }
1446
1447         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1448             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1449         } else {
1450             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451                 .",lun=$drive->{index}";
1452         }
1453         $device .= ",drive=drive-$drive_id,id=$drive_id";
1454
1455         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456             $device .= ",rotation_rate=1";
1457         }
1458         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1459
1460     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1461         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1462         my $controller = int($drive->{index} / $maxdev);
1463         my $unit = $drive->{index} % $maxdev;
1464         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1465
1466         $device = "ide-$devicetype";
1467         if ($drive->{interface} eq 'ide') {
1468             $device .= ",bus=ide.$controller,unit=$unit";
1469         } else {
1470             $device .= ",bus=ahci$controller.$unit";
1471         }
1472         $device .= ",drive=drive-$drive_id,id=$drive_id";
1473
1474         if ($devicetype eq 'hd') {
1475             if (my $model = $drive->{model}) {
1476                 $model = URI::Escape::uri_unescape($model);
1477                 $device .= ",model=$model";
1478             }
1479             if ($drive->{ssd}) {
1480                 $device .= ",rotation_rate=1";
1481             }
1482         }
1483         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1484     } elsif ($drive->{interface} eq 'usb') {
1485         die "implement me";
1486         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1487     } else {
1488         die "unsupported interface type";
1489     }
1490
1491     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1492
1493     if (my $serial = $drive->{serial}) {
1494         $serial = URI::Escape::uri_unescape($serial);
1495         $device .= ",serial=$serial";
1496     }
1497
1498
1499     return $device;
1500 }
1501
1502 sub get_initiator_name {
1503     my $initiator;
1504
1505     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1506     while (defined(my $line = <$fh>)) {
1507         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1508         $initiator = $1;
1509         last;
1510     }
1511     $fh->close();
1512
1513     return $initiator;
1514 }
1515
1516 sub print_drive_commandline_full {
1517     my ($storecfg, $vmid, $drive) = @_;
1518
1519     my $path;
1520     my $volid = $drive->{file};
1521     my $format;
1522
1523     if (drive_is_cdrom($drive)) {
1524         $path = get_iso_path($storecfg, $vmid, $volid);
1525     } else {
1526         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1527         if ($storeid) {
1528             $path = PVE::Storage::path($storecfg, $volid);
1529             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1530             $format = qemu_img_format($scfg, $volname);
1531         } else {
1532             $path = $volid;
1533             $format = "raw";
1534         }
1535    }
1536
1537     my $opts = '';
1538     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539     foreach my $o (@qemu_drive_options) {
1540         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1541     }
1542
1543     # snapshot only accepts on|off
1544     if (defined($drive->{snapshot})) {
1545         my $v = $drive->{snapshot} ? 'on' : 'off';
1546         $opts .= ",snapshot=$v";
1547     }
1548
1549     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1550         my ($dir, $qmpname) = @$type;
1551         if (my $v = $drive->{"mbps$dir"}) {
1552             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1553         }
1554         if (my $v = $drive->{"mbps${dir}_max"}) {
1555             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1556         }
1557         if (my $v = $drive->{"bps${dir}_max_length"}) {
1558             $opts .= ",throttling.bps$qmpname-max-length=$v";
1559         }
1560         if (my $v = $drive->{"iops${dir}"}) {
1561             $opts .= ",throttling.iops$qmpname=$v";
1562         }
1563         if (my $v = $drive->{"iops${dir}_max"}) {
1564             $opts .= ",throttling.iops$qmpname-max=$v";
1565         }
1566         if (my $v = $drive->{"iops${dir}_max_length"}) {
1567             $opts .= ",throttling.iops$qmpname-max-length=$v";
1568         }
1569     }
1570
1571     $opts .= ",format=$format" if $format && !$drive->{format};
1572
1573     my $cache_direct = 0;
1574
1575     if (my $cache = $drive->{cache}) {
1576         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577     } elsif (!drive_is_cdrom($drive)) {
1578         $opts .= ",cache=none";
1579         $cache_direct = 1;
1580     }
1581
1582     # aio native works only with O_DIRECT
1583     if (!$drive->{aio}) {
1584         if($cache_direct) {
1585             $opts .= ",aio=native";
1586         } else {
1587             $opts .= ",aio=threads";
1588         }
1589     }
1590
1591     if (!drive_is_cdrom($drive)) {
1592         my $detectzeroes;
1593         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1594             $detectzeroes = 'off';
1595         } elsif ($drive->{discard}) {
1596             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1597         } else {
1598             # This used to be our default with discard not being specified:
1599             $detectzeroes = 'on';
1600         }
1601         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1602     }
1603
1604     my $pathinfo = $path ? "file=$path," : '';
1605
1606     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1607 }
1608
1609 sub print_netdevice_full {
1610     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1611
1612     my $device = $net->{model};
1613     if ($net->{model} eq 'virtio') {
1614          $device = 'virtio-net-pci';
1615      };
1616
1617     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1618     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1620         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621         # and out of each queue plus one config interrupt and control vector queue
1622         my $vectors = $net->{queues} * 2 + 2;
1623         $tmpstr .= ",vectors=$vectors,mq=on";
1624     }
1625     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1626
1627     if (my $mtu = $net->{mtu}) {
1628         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1629             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1630             if ($mtu == 1) {
1631                  $mtu = $bridge_mtu;
1632             } elsif ($mtu < 576) {
1633                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634             } elsif ($mtu > $bridge_mtu) {
1635                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1636             }
1637             $tmpstr .= ",host_mtu=$mtu";
1638         } else {
1639             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1640         }
1641     }
1642
1643     if ($use_old_bios_files) {
1644         my $romfile;
1645         if ($device eq 'virtio-net-pci') {
1646             $romfile = 'pxe-virtio.rom';
1647         } elsif ($device eq 'e1000') {
1648             $romfile = 'pxe-e1000.rom';
1649         } elsif ($device eq 'ne2k') {
1650             $romfile = 'pxe-ne2k_pci.rom';
1651         } elsif ($device eq 'pcnet') {
1652             $romfile = 'pxe-pcnet.rom';
1653         } elsif ($device eq 'rtl8139') {
1654             $romfile = 'pxe-rtl8139.rom';
1655         }
1656         $tmpstr .= ",romfile=$romfile" if $romfile;
1657     }
1658
1659     return $tmpstr;
1660 }
1661
1662 sub print_netdev_full {
1663     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1664
1665     my $i = '';
1666     if ($netid =~ m/^net(\d+)$/) {
1667         $i = int($1);
1668     }
1669
1670     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1671
1672     my $ifname = "tap${vmid}i$i";
1673
1674     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675     die "interface name '$ifname' is too long (max 15 character)\n"
1676         if length($ifname) >= 16;
1677
1678     my $vhostparam = '';
1679     if (is_native($arch)) {
1680         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1681     }
1682
1683     my $vmname = $conf->{name} || "vm$vmid";
1684
1685     my $netdev = "";
1686     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1687
1688     if ($net->{bridge}) {
1689         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1691     } else {
1692         $netdev = "type=user,id=$netid,hostname=$vmname";
1693     }
1694
1695     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1696
1697     return $netdev;
1698 }
1699
1700 my $vga_map = {
1701     'cirrus' => 'cirrus-vga',
1702     'std' => 'VGA',
1703     'vmware' => 'vmware-svga',
1704     'virtio' => 'virtio-vga',
1705 };
1706
1707 sub print_vga_device {
1708     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1709
1710     my $type = $vga_map->{$vga->{type}};
1711     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712         $type = 'virtio-gpu';
1713     }
1714     my $vgamem_mb = $vga->{memory};
1715
1716     my $max_outputs = '';
1717     if ($qxlnum) {
1718         $type = $id ? 'qxl' : 'qxl-vga';
1719
1720         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1721             # set max outputs so linux can have up to 4 qxl displays with one device
1722             if (min_version($machine_version, 4, 1)) {
1723                 $max_outputs = ",max_outputs=4";
1724             }
1725         }
1726     }
1727
1728     die "no devicetype for $vga->{type}\n" if !$type;
1729
1730     my $memory = "";
1731     if ($vgamem_mb) {
1732         if ($vga->{type} eq 'virtio') {
1733             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1734             $memory = ",max_hostmem=$bytes";
1735         } elsif ($qxlnum) {
1736             # from https://www.spice-space.org/multiple-monitors.html
1737             $memory = ",vgamem_mb=$vga->{memory}";
1738             my $ram = $vgamem_mb * 4;
1739             my $vram = $vgamem_mb * 2;
1740             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1741         } else {
1742             $memory = ",vgamem_mb=$vga->{memory}";
1743         }
1744     } elsif ($qxlnum && $id) {
1745         $memory = ",ram_size=67108864,vram_size=33554432";
1746     }
1747
1748     my $edidoff = "";
1749     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1750         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1751     }
1752
1753     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1754     my $vgaid = "vga" . ($id // '');
1755     my $pciaddr;
1756
1757     if ($q35 && $vgaid eq 'vga') {
1758         # the first display uses pcie.0 bus on q35 machines
1759         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1760     } else {
1761         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1762     }
1763
1764     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1765 }
1766
1767 sub parse_number_sets {
1768     my ($set) = @_;
1769     my $res = [];
1770     foreach my $part (split(/;/, $set)) {
1771         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773             push @$res, [ $1, $2 ];
1774         } else {
1775             die "invalid range: $part\n";
1776         }
1777     }
1778     return $res;
1779 }
1780
1781 sub parse_numa {
1782     my ($data) = @_;
1783
1784     my $res = parse_property_string($numa_fmt, $data);
1785     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1786     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1787     return $res;
1788 }
1789
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1791 sub parse_net {
1792     my ($data) = @_;
1793
1794     my $res = eval { parse_property_string($net_fmt, $data) };
1795     if ($@) {
1796         warn $@;
1797         return;
1798     }
1799     if (!defined($res->{macaddr})) {
1800         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1801         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1802     }
1803     return $res;
1804 }
1805
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig {
1808     my ($data) = @_;
1809
1810     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1811     if ($@) {
1812         warn $@;
1813         return;
1814     }
1815
1816     if ($res->{gw} && !$res->{ip}) {
1817         warn 'gateway specified without specifying an IP address';
1818         return;
1819     }
1820     if ($res->{gw6} && !$res->{ip6}) {
1821         warn 'IPv6 gateway specified without specifying an IPv6 address';
1822         return;
1823     }
1824     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1825         warn 'gateway specified together with DHCP';
1826         return;
1827     }
1828     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1829         # gw6 + auto/dhcp
1830         warn "IPv6 gateway specified together with $res->{ip6} address";
1831         return;
1832     }
1833
1834     if (!$res->{ip} && !$res->{ip6}) {
1835         return { ip => 'dhcp', ip6 => 'dhcp' };
1836     }
1837
1838     return $res;
1839 }
1840
1841 sub print_net {
1842     my $net = shift;
1843
1844     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1845 }
1846
1847 sub add_random_macs {
1848     my ($settings) = @_;
1849
1850     foreach my $opt (keys %$settings) {
1851         next if $opt !~ m/^net(\d+)$/;
1852         my $net = parse_net($settings->{$opt});
1853         next if !$net;
1854         $settings->{$opt} = print_net($net);
1855     }
1856 }
1857
1858 sub vm_is_volid_owner {
1859     my ($storecfg, $vmid, $volid) = @_;
1860
1861     if ($volid !~  m|^/|) {
1862         my ($path, $owner);
1863         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1864         if ($owner && ($owner == $vmid)) {
1865             return 1;
1866         }
1867     }
1868
1869     return;
1870 }
1871
1872 sub vmconfig_register_unused_drive {
1873     my ($storecfg, $vmid, $conf, $drive) = @_;
1874
1875     if (drive_is_cloudinit($drive)) {
1876         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1877         warn $@ if $@;
1878     } elsif (!drive_is_cdrom($drive)) {
1879         my $volid = $drive->{file};
1880         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1881             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1882         }
1883     }
1884 }
1885
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1887 my $smbios1_fmt = {
1888     uuid => {
1889         type => 'string',
1890         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891         format_description => 'UUID',
1892         description => "Set SMBIOS1 UUID.",
1893         optional => 1,
1894     },
1895     version => {
1896         type => 'string',
1897         pattern => '[A-Za-z0-9+\/]+={0,2}',
1898         format_description => 'Base64 encoded string',
1899         description => "Set SMBIOS1 version.",
1900         optional => 1,
1901     },
1902     serial => {
1903         type => 'string',
1904         pattern => '[A-Za-z0-9+\/]+={0,2}',
1905         format_description => 'Base64 encoded string',
1906         description => "Set SMBIOS1 serial number.",
1907         optional => 1,
1908     },
1909     manufacturer => {
1910         type => 'string',
1911         pattern => '[A-Za-z0-9+\/]+={0,2}',
1912         format_description => 'Base64 encoded string',
1913         description => "Set SMBIOS1 manufacturer.",
1914         optional => 1,
1915     },
1916     product => {
1917         type => 'string',
1918         pattern => '[A-Za-z0-9+\/]+={0,2}',
1919         format_description => 'Base64 encoded string',
1920         description => "Set SMBIOS1 product ID.",
1921         optional => 1,
1922     },
1923     sku => {
1924         type => 'string',
1925         pattern => '[A-Za-z0-9+\/]+={0,2}',
1926         format_description => 'Base64 encoded string',
1927         description => "Set SMBIOS1 SKU string.",
1928         optional => 1,
1929     },
1930     family => {
1931         type => 'string',
1932         pattern => '[A-Za-z0-9+\/]+={0,2}',
1933         format_description => 'Base64 encoded string',
1934         description => "Set SMBIOS1 family string.",
1935         optional => 1,
1936     },
1937     base64 => {
1938         type => 'boolean',
1939         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1940         optional => 1,
1941     },
1942 };
1943
1944 sub parse_smbios1 {
1945     my ($data) = @_;
1946
1947     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1948     warn $@ if $@;
1949     return $res;
1950 }
1951
1952 sub print_smbios1 {
1953     my ($smbios1) = @_;
1954     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1955 }
1956
1957 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1958
1959 sub parse_watchdog {
1960     my ($value) = @_;
1961
1962     return if !$value;
1963
1964     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1965     warn $@ if $@;
1966     return $res;
1967 }
1968
1969 sub parse_guest_agent {
1970     my ($conf) = @_;
1971
1972     return {} if !defined($conf->{agent});
1973
1974     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
1975     warn $@ if $@;
1976
1977     # if the agent is disabled ignore the other potentially set properties
1978     return {} if !$res->{enabled};
1979     return $res;
1980 }
1981
1982 sub get_qga_key {
1983     my ($conf, $key) = @_;
1984     return undef if !defined($conf->{agent});
1985
1986     my $agent = parse_guest_agent($conf);
1987     return $agent->{$key};
1988 }
1989
1990 sub parse_vga {
1991     my ($value) = @_;
1992
1993     return {} if !$value;
1994     my $res = eval { parse_property_string($vga_fmt, $value) };
1995     warn $@ if $@;
1996     return $res;
1997 }
1998
1999 sub parse_rng {
2000     my ($value) = @_;
2001
2002     return if !$value;
2003
2004     my $res = eval { parse_property_string($rng_fmt, $value) };
2005     warn $@ if $@;
2006     return $res;
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2010 sub verify_usb_device {
2011     my ($value, $noerr) = @_;
2012
2013     return $value if parse_usb_device($value);
2014
2015     return if $noerr;
2016
2017     die "unable to parse usb device\n";
2018 }
2019
2020 # add JSON properties for create and set function
2021 sub json_config_properties {
2022     my $prop = shift;
2023
2024     foreach my $opt (keys %$confdesc) {
2025         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027         $prop->{$opt} = $confdesc->{$opt};
2028     }
2029
2030     return $prop;
2031 }
2032
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties {
2035
2036     return dclone($confdesc_cloudinit);
2037 }
2038
2039 sub check_type {
2040     my ($key, $value) = @_;
2041
2042     die "unknown setting '$key'\n" if !$confdesc->{$key};
2043
2044     my $type = $confdesc->{$key}->{type};
2045
2046     if (!defined($value)) {
2047         die "got undefined value\n";
2048     }
2049
2050     if ($value =~ m/[\n\r]/) {
2051         die "property contains a line feed\n";
2052     }
2053
2054     if ($type eq 'boolean') {
2055         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057         die "type check ('boolean') failed - got '$value'\n";
2058     } elsif ($type eq 'integer') {
2059         return int($1) if $value =~ m/^(\d+)$/;
2060         die "type check ('integer') failed - got '$value'\n";
2061     } elsif ($type eq 'number') {
2062         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063         die "type check ('number') failed - got '$value'\n";
2064     } elsif ($type eq 'string') {
2065         if (my $fmt = $confdesc->{$key}->{format}) {
2066             PVE::JSONSchema::check_format($fmt, $value);
2067             return $value;
2068         }
2069         $value =~ s/^\"(.*)\"$/$1/;
2070         return $value;
2071     } else {
2072         die "internal error"
2073     }
2074 }
2075
2076 sub destroy_vm {
2077     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2078
2079     my $conf = PVE::QemuConfig->load_config($vmid);
2080
2081     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2082
2083     if ($conf->{template}) {
2084         # check if any base image is still used by a linked clone
2085         PVE::QemuConfig->foreach_volume($conf, sub {
2086                 my ($ds, $drive) = @_;
2087                 return if drive_is_cdrom($drive);
2088
2089                 my $volid = $drive->{file};
2090                 return if !$volid || $volid =~ m|^/|;
2091
2092                 die "base volume '$volid' is still in use by linked cloned\n"
2093                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2094
2095         });
2096     }
2097
2098     # only remove disks owned by this VM
2099     PVE::QemuConfig->foreach_volume($conf, sub {
2100         my ($ds, $drive) = @_;
2101         return if drive_is_cdrom($drive, 1);
2102
2103         my $volid = $drive->{file};
2104         return if !$volid || $volid =~ m|^/|;
2105
2106         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2107         return if !$path || !$owner || ($owner != $vmid);
2108
2109         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2110         warn "Could not remove disk '$volid', check manually: $@" if $@;
2111     });
2112
2113     # also remove unused disk
2114     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2115     PVE::Storage::foreach_volid($vmdisks, sub {
2116         my ($volid, $sid, $volname, $d) = @_;
2117         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2118         warn $@ if $@;
2119     });
2120
2121     if (defined $replacement_conf) {
2122         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2123     } else {
2124         PVE::QemuConfig->destroy_config($vmid);
2125     }
2126 }
2127
2128 sub parse_vm_config {
2129     my ($filename, $raw) = @_;
2130
2131     return if !defined($raw);
2132
2133     my $res = {
2134         digest => Digest::SHA::sha1_hex($raw),
2135         snapshots => {},
2136         pending => {},
2137     };
2138
2139     $filename =~ m|/qemu-server/(\d+)\.conf$|
2140         || die "got strange filename '$filename'";
2141
2142     my $vmid = $1;
2143
2144     my $conf = $res;
2145     my $descr;
2146     my $section = '';
2147
2148     my @lines = split(/\n/, $raw);
2149     foreach my $line (@lines) {
2150         next if $line =~ m/^\s*$/;
2151
2152         if ($line =~ m/^\[PENDING\]\s*$/i) {
2153             $section = 'pending';
2154             if (defined($descr)) {
2155                 $descr =~ s/\s+$//;
2156                 $conf->{description} = $descr;
2157             }
2158             $descr = undef;
2159             $conf = $res->{$section} = {};
2160             next;
2161
2162         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2163             $section = $1;
2164             if (defined($descr)) {
2165                 $descr =~ s/\s+$//;
2166                 $conf->{description} = $descr;
2167             }
2168             $descr = undef;
2169             $conf = $res->{snapshots}->{$section} = {};
2170             next;
2171         }
2172
2173         if ($line =~ m/^\#(.*)\s*$/) {
2174             $descr = '' if !defined($descr);
2175             $descr .= PVE::Tools::decode_text($1) . "\n";
2176             next;
2177         }
2178
2179         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2180             $descr = '' if !defined($descr);
2181             $descr .= PVE::Tools::decode_text($2);
2182         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2183             $conf->{snapstate} = $1;
2184         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2185             my $key = $1;
2186             my $value = $2;
2187             $conf->{$key} = $value;
2188         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2189             my $value = $1;
2190             if ($section eq 'pending') {
2191                 $conf->{delete} = $value; # we parse this later
2192             } else {
2193                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2194             }
2195         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2196             my $key = $1;
2197             my $value = $2;
2198             eval { $value = check_type($key, $value); };
2199             if ($@) {
2200                 warn "vm $vmid - unable to parse value of '$key' - $@";
2201             } else {
2202                 $key = 'ide2' if $key eq 'cdrom';
2203                 my $fmt = $confdesc->{$key}->{format};
2204                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2205                     my $v = parse_drive($key, $value);
2206                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2207                         $v->{file} = $volid;
2208                         $value = print_drive($v);
2209                     } else {
2210                         warn "vm $vmid - unable to parse value of '$key'\n";
2211                         next;
2212                     }
2213                 }
2214
2215                 $conf->{$key} = $value;
2216             }
2217         }
2218     }
2219
2220     if (defined($descr)) {
2221         $descr =~ s/\s+$//;
2222         $conf->{description} = $descr;
2223     }
2224     delete $res->{snapstate}; # just to be sure
2225
2226     return $res;
2227 }
2228
2229 sub write_vm_config {
2230     my ($filename, $conf) = @_;
2231
2232     delete $conf->{snapstate}; # just to be sure
2233
2234     if ($conf->{cdrom}) {
2235         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2236         $conf->{ide2} = $conf->{cdrom};
2237         delete $conf->{cdrom};
2238     }
2239
2240     # we do not use 'smp' any longer
2241     if ($conf->{sockets}) {
2242         delete $conf->{smp};
2243     } elsif ($conf->{smp}) {
2244         $conf->{sockets} = $conf->{smp};
2245         delete $conf->{cores};
2246         delete $conf->{smp};
2247     }
2248
2249     my $used_volids = {};
2250
2251     my $cleanup_config = sub {
2252         my ($cref, $pending, $snapname) = @_;
2253
2254         foreach my $key (keys %$cref) {
2255             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2256                 $key eq 'snapstate' || $key eq 'pending';
2257             my $value = $cref->{$key};
2258             if ($key eq 'delete') {
2259                 die "propertry 'delete' is only allowed in [PENDING]\n"
2260                     if !$pending;
2261                 # fixme: check syntax?
2262                 next;
2263             }
2264             eval { $value = check_type($key, $value); };
2265             die "unable to parse value of '$key' - $@" if $@;
2266
2267             $cref->{$key} = $value;
2268
2269             if (!$snapname && is_valid_drivename($key)) {
2270                 my $drive = parse_drive($key, $value);
2271                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2272             }
2273         }
2274     };
2275
2276     &$cleanup_config($conf);
2277
2278     &$cleanup_config($conf->{pending}, 1);
2279
2280     foreach my $snapname (keys %{$conf->{snapshots}}) {
2281         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2282         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2283     }
2284
2285     # remove 'unusedX' settings if we re-add a volume
2286     foreach my $key (keys %$conf) {
2287         my $value = $conf->{$key};
2288         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2289             delete $conf->{$key};
2290         }
2291     }
2292
2293     my $generate_raw_config = sub {
2294         my ($conf, $pending) = @_;
2295
2296         my $raw = '';
2297
2298         # add description as comment to top of file
2299         if (defined(my $descr = $conf->{description})) {
2300             if ($descr) {
2301                 foreach my $cl (split(/\n/, $descr)) {
2302                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2303                 }
2304             } else {
2305                 $raw .= "#\n" if $pending;
2306             }
2307         }
2308
2309         foreach my $key (sort keys %$conf) {
2310             next if $key =~ /^(digest|description|pending|snapshots)$/;
2311             $raw .= "$key: $conf->{$key}\n";
2312         }
2313         return $raw;
2314     };
2315
2316     my $raw = &$generate_raw_config($conf);
2317
2318     if (scalar(keys %{$conf->{pending}})){
2319         $raw .= "\n[PENDING]\n";
2320         $raw .= &$generate_raw_config($conf->{pending}, 1);
2321     }
2322
2323     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2324         $raw .= "\n[$snapname]\n";
2325         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2326     }
2327
2328     return $raw;
2329 }
2330
2331 sub load_defaults {
2332
2333     my $res = {};
2334
2335     # we use static defaults from our JSON schema configuration
2336     foreach my $key (keys %$confdesc) {
2337         if (defined(my $default = $confdesc->{$key}->{default})) {
2338             $res->{$key} = $default;
2339         }
2340     }
2341
2342     return $res;
2343 }
2344
2345 sub config_list {
2346     my $vmlist = PVE::Cluster::get_vmlist();
2347     my $res = {};
2348     return $res if !$vmlist || !$vmlist->{ids};
2349     my $ids = $vmlist->{ids};
2350     my $nodename = nodename();
2351
2352     foreach my $vmid (keys %$ids) {
2353         my $d = $ids->{$vmid};
2354         next if !$d->{node} || $d->{node} ne $nodename;
2355         next if !$d->{type} || $d->{type} ne 'qemu';
2356         $res->{$vmid}->{exists} = 1;
2357     }
2358     return $res;
2359 }
2360
2361 # test if VM uses local resources (to prevent migration)
2362 sub check_local_resources {
2363     my ($conf, $noerr) = @_;
2364
2365     my @loc_res = ();
2366
2367     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2368     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2369
2370     push @loc_res, "ivshmem" if $conf->{ivshmem};
2371
2372     foreach my $k (keys %$conf) {
2373         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2374         # sockets are safe: they will recreated be on the target side post-migrate
2375         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2376         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2377     }
2378
2379     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2380
2381     return \@loc_res;
2382 }
2383
2384 # check if used storages are available on all nodes (use by migrate)
2385 sub check_storage_availability {
2386     my ($storecfg, $conf, $node) = @_;
2387
2388     PVE::QemuConfig->foreach_volume($conf, sub {
2389         my ($ds, $drive) = @_;
2390
2391         my $volid = $drive->{file};
2392         return if !$volid;
2393
2394         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2395         return if !$sid;
2396
2397         # check if storage is available on both nodes
2398         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2399         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2400    });
2401 }
2402
2403 # list nodes where all VM images are available (used by has_feature API)
2404 sub shared_nodes {
2405     my ($conf, $storecfg) = @_;
2406
2407     my $nodelist = PVE::Cluster::get_nodelist();
2408     my $nodehash = { map { $_ => 1 } @$nodelist };
2409     my $nodename = nodename();
2410
2411     PVE::QemuConfig->foreach_volume($conf, sub {
2412         my ($ds, $drive) = @_;
2413
2414         my $volid = $drive->{file};
2415         return if !$volid;
2416
2417         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2418         if ($storeid) {
2419             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2420             if ($scfg->{disable}) {
2421                 $nodehash = {};
2422             } elsif (my $avail = $scfg->{nodes}) {
2423                 foreach my $node (keys %$nodehash) {
2424                     delete $nodehash->{$node} if !$avail->{$node};
2425                 }
2426             } elsif (!$scfg->{shared}) {
2427                 foreach my $node (keys %$nodehash) {
2428                     delete $nodehash->{$node} if $node ne $nodename
2429                 }
2430             }
2431         }
2432     });
2433
2434     return $nodehash
2435 }
2436
2437 sub check_local_storage_availability {
2438     my ($conf, $storecfg) = @_;
2439
2440     my $nodelist = PVE::Cluster::get_nodelist();
2441     my $nodehash = { map { $_ => {} } @$nodelist };
2442
2443     PVE::QemuConfig->foreach_volume($conf, sub {
2444         my ($ds, $drive) = @_;
2445
2446         my $volid = $drive->{file};
2447         return if !$volid;
2448
2449         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2450         if ($storeid) {
2451             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2452
2453             if ($scfg->{disable}) {
2454                 foreach my $node (keys %$nodehash) {
2455                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2456                 }
2457             } elsif (my $avail = $scfg->{nodes}) {
2458                 foreach my $node (keys %$nodehash) {
2459                     if (!$avail->{$node}) {
2460                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2461                     }
2462                 }
2463             }
2464         }
2465     });
2466
2467     foreach my $node (values %$nodehash) {
2468         if (my $unavail = $node->{unavailable_storages}) {
2469             $node->{unavailable_storages} = [ sort keys %$unavail ];
2470         }
2471     }
2472
2473     return $nodehash
2474 }
2475
2476 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2477 sub check_running {
2478     my ($vmid, $nocheck, $node) = @_;
2479
2480     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2481     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2482 }
2483
2484 sub vzlist {
2485
2486     my $vzlist = config_list();
2487
2488     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2489
2490     while (defined(my $de = $fd->read)) {
2491         next if $de !~ m/^(\d+)\.pid$/;
2492         my $vmid = $1;
2493         next if !defined($vzlist->{$vmid});
2494         if (my $pid = check_running($vmid)) {
2495             $vzlist->{$vmid}->{pid} = $pid;
2496         }
2497     }
2498
2499     return $vzlist;
2500 }
2501
2502 our $vmstatus_return_properties = {
2503     vmid => get_standard_option('pve-vmid'),
2504     status => {
2505         description => "Qemu process status.",
2506         type => 'string',
2507         enum => ['stopped', 'running'],
2508     },
2509     maxmem => {
2510         description => "Maximum memory in bytes.",
2511         type => 'integer',
2512         optional => 1,
2513         renderer => 'bytes',
2514     },
2515     maxdisk => {
2516         description => "Root disk size in bytes.",
2517         type => 'integer',
2518         optional => 1,
2519         renderer => 'bytes',
2520     },
2521     name => {
2522         description => "VM name.",
2523         type => 'string',
2524         optional => 1,
2525     },
2526     qmpstatus => {
2527         description => "Qemu QMP agent status.",
2528         type => 'string',
2529         optional => 1,
2530     },
2531     pid => {
2532         description => "PID of running qemu process.",
2533         type => 'integer',
2534         optional => 1,
2535     },
2536     uptime => {
2537         description => "Uptime.",
2538         type => 'integer',
2539         optional => 1,
2540         renderer => 'duration',
2541     },
2542     cpus => {
2543         description => "Maximum usable CPUs.",
2544         type => 'number',
2545         optional => 1,
2546     },
2547     lock => {
2548         description => "The current config lock, if any.",
2549         type => 'string',
2550         optional => 1,
2551     },
2552     tags => {
2553         description  => "The current configured tags, if any",
2554         type => 'string',
2555         optional => 1,
2556     },
2557 };
2558
2559 my $last_proc_pid_stat;
2560
2561 # get VM status information
2562 # This must be fast and should not block ($full == false)
2563 # We only query KVM using QMP if $full == true (this can be slow)
2564 sub vmstatus {
2565     my ($opt_vmid, $full) = @_;
2566
2567     my $res = {};
2568
2569     my $storecfg = PVE::Storage::config();
2570
2571     my $list = vzlist();
2572     my $defaults = load_defaults();
2573
2574     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2575
2576     my $cpucount = $cpuinfo->{cpus} || 1;
2577
2578     foreach my $vmid (keys %$list) {
2579         next if $opt_vmid && ($vmid ne $opt_vmid);
2580
2581         my $conf = PVE::QemuConfig->load_config($vmid);
2582
2583         my $d = { vmid => $vmid };
2584         $d->{pid} = $list->{$vmid}->{pid};
2585
2586         # fixme: better status?
2587         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2588
2589         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2590         if (defined($size)) {
2591             $d->{disk} = 0; # no info available
2592             $d->{maxdisk} = $size;
2593         } else {
2594             $d->{disk} = 0;
2595             $d->{maxdisk} = 0;
2596         }
2597
2598         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2599             * ($conf->{cores} || $defaults->{cores});
2600         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2601         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2602
2603         $d->{name} = $conf->{name} || "VM $vmid";
2604         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2605             : $defaults->{memory}*(1024*1024);
2606
2607         if ($conf->{balloon}) {
2608             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2609             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2610                 : $defaults->{shares};
2611         }
2612
2613         $d->{uptime} = 0;
2614         $d->{cpu} = 0;
2615         $d->{mem} = 0;
2616
2617         $d->{netout} = 0;
2618         $d->{netin} = 0;
2619
2620         $d->{diskread} = 0;
2621         $d->{diskwrite} = 0;
2622
2623         $d->{template} = PVE::QemuConfig->is_template($conf);
2624
2625         $d->{serial} = 1 if conf_has_serial($conf);
2626         $d->{lock} = $conf->{lock} if $conf->{lock};
2627         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2628
2629         $res->{$vmid} = $d;
2630     }
2631
2632     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2633     foreach my $dev (keys %$netdev) {
2634         next if $dev !~ m/^tap([1-9]\d*)i/;
2635         my $vmid = $1;
2636         my $d = $res->{$vmid};
2637         next if !$d;
2638
2639         $d->{netout} += $netdev->{$dev}->{receive};
2640         $d->{netin} += $netdev->{$dev}->{transmit};
2641
2642         if ($full) {
2643             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2644             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2645         }
2646
2647     }
2648
2649     my $ctime = gettimeofday;
2650
2651     foreach my $vmid (keys %$list) {
2652
2653         my $d = $res->{$vmid};
2654         my $pid = $d->{pid};
2655         next if !$pid;
2656
2657         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2658         next if !$pstat; # not running
2659
2660         my $used = $pstat->{utime} + $pstat->{stime};
2661
2662         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2663
2664         if ($pstat->{vsize}) {
2665             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2666         }
2667
2668         my $old = $last_proc_pid_stat->{$pid};
2669         if (!$old) {
2670             $last_proc_pid_stat->{$pid} = {
2671                 time => $ctime,
2672                 used => $used,
2673                 cpu => 0,
2674             };
2675             next;
2676         }
2677
2678         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2679
2680         if ($dtime > 1000) {
2681             my $dutime = $used -  $old->{used};
2682
2683             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2684             $last_proc_pid_stat->{$pid} = {
2685                 time => $ctime,
2686                 used => $used,
2687                 cpu => $d->{cpu},
2688             };
2689         } else {
2690             $d->{cpu} = $old->{cpu};
2691         }
2692     }
2693
2694     return $res if !$full;
2695
2696     my $qmpclient = PVE::QMPClient->new();
2697
2698     my $ballooncb = sub {
2699         my ($vmid, $resp) = @_;
2700
2701         my $info = $resp->{'return'};
2702         return if !$info->{max_mem};
2703
2704         my $d = $res->{$vmid};
2705
2706         # use memory assigned to VM
2707         $d->{maxmem} = $info->{max_mem};
2708         $d->{balloon} = $info->{actual};
2709
2710         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2711             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2712             $d->{freemem} = $info->{free_mem};
2713         }
2714
2715         $d->{ballooninfo} = $info;
2716     };
2717
2718     my $blockstatscb = sub {
2719         my ($vmid, $resp) = @_;
2720         my $data = $resp->{'return'} || [];
2721         my $totalrdbytes = 0;
2722         my $totalwrbytes = 0;
2723
2724         for my $blockstat (@$data) {
2725             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2726             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2727
2728             $blockstat->{device} =~ s/drive-//;
2729             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2730         }
2731         $res->{$vmid}->{diskread} = $totalrdbytes;
2732         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2733     };
2734
2735     my $statuscb = sub {
2736         my ($vmid, $resp) = @_;
2737
2738         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2739         # this fails if ballon driver is not loaded, so this must be
2740         # the last commnand (following command are aborted if this fails).
2741         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2742
2743         my $status = 'unknown';
2744         if (!defined($status = $resp->{'return'}->{status})) {
2745             warn "unable to get VM status\n";
2746             return;
2747         }
2748
2749         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2750     };
2751
2752     foreach my $vmid (keys %$list) {
2753         next if $opt_vmid && ($vmid ne $opt_vmid);
2754         next if !$res->{$vmid}->{pid}; # not running
2755         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2756     }
2757
2758     $qmpclient->queue_execute(undef, 2);
2759
2760     foreach my $vmid (keys %$list) {
2761         next if $opt_vmid && ($vmid ne $opt_vmid);
2762         next if !$res->{$vmid}->{pid}; #not running
2763
2764         # we can't use the $qmpclient since it might have already aborted on
2765         # 'query-balloon', but this might also fail for older versions...
2766         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2767         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2768     }
2769
2770     foreach my $vmid (keys %$list) {
2771         next if $opt_vmid && ($vmid ne $opt_vmid);
2772         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2773     }
2774
2775     return $res;
2776 }
2777
2778 sub conf_has_serial {
2779     my ($conf) = @_;
2780
2781     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2782         if ($conf->{"serial$i"}) {
2783             return 1;
2784         }
2785     }
2786
2787     return 0;
2788 }
2789
2790 sub conf_has_audio {
2791     my ($conf, $id) = @_;
2792
2793     $id //= 0;
2794     my $audio = $conf->{"audio$id"};
2795     return if !defined($audio);
2796
2797     my $audioproperties = parse_property_string($audio_fmt, $audio);
2798     my $audiodriver = $audioproperties->{driver} // 'spice';
2799
2800     return {
2801         dev => $audioproperties->{device},
2802         dev_id => "audiodev$id",
2803         backend => $audiodriver,
2804         backend_id => "$audiodriver-backend${id}",
2805     };
2806 }
2807
2808 sub audio_devs {
2809     my ($audio, $audiopciaddr, $machine_version) = @_;
2810
2811     my $devs = [];
2812
2813     my $id = $audio->{dev_id};
2814     my $audiodev = "";
2815     if (min_version($machine_version, 4, 2)) {
2816         $audiodev = ",audiodev=$audio->{backend_id}";
2817     }
2818
2819     if ($audio->{dev} eq 'AC97') {
2820         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2821     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2822         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2823         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2824         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2825     } else {
2826         die "unkown audio device '$audio->{dev}', implement me!";
2827     }
2828
2829     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2830
2831     return $devs;
2832 }
2833
2834 sub vga_conf_has_spice {
2835     my ($vga) = @_;
2836
2837     my $vgaconf = parse_vga($vga);
2838     my $vgatype = $vgaconf->{type};
2839     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2840
2841     return $1 || 1;
2842 }
2843
2844 sub is_native($) {
2845     my ($arch) = @_;
2846     return get_host_arch() eq $arch;
2847 }
2848
2849 sub get_vm_arch {
2850     my ($conf) = @_;
2851     return $conf->{arch} // get_host_arch();
2852 }
2853
2854 my $default_machines = {
2855     x86_64 => 'pc',
2856     aarch64 => 'virt',
2857 };
2858
2859 sub get_vm_machine {
2860     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2861
2862     my $machine = $forcemachine || $conf->{machine};
2863
2864     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2865         $arch //= 'x86_64';
2866         $machine ||= $default_machines->{$arch};
2867         if ($add_pve_version) {
2868             $kvmversion //= kvm_user_version();
2869             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2870             $machine .= "+pve$pvever";
2871         }
2872     }
2873
2874     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2875         # for version-pinned machines that do not include a pve-version (e.g.
2876         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2877         $machine .= '+pve0';
2878     }
2879
2880     return $machine;
2881 }
2882
2883 sub get_ovmf_files($) {
2884     my ($arch) = @_;
2885
2886     my $ovmf = $OVMF->{$arch}
2887         or die "no OVMF images known for architecture '$arch'\n";
2888
2889     return @$ovmf;
2890 }
2891
2892 my $Arch2Qemu = {
2893     aarch64 => '/usr/bin/qemu-system-aarch64',
2894     x86_64 => '/usr/bin/qemu-system-x86_64',
2895 };
2896 sub get_command_for_arch($) {
2897     my ($arch) = @_;
2898     return '/usr/bin/kvm' if is_native($arch);
2899
2900     my $cmd = $Arch2Qemu->{$arch}
2901         or die "don't know how to emulate architecture '$arch'\n";
2902     return $cmd;
2903 }
2904
2905 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2906 # to use in a QEMU command line (-cpu element), first array_intersect the result
2907 # of query_supported_ with query_understood_. This is necessary because:
2908 #
2909 # a) query_understood_ returns flags the host cannot use and
2910 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2911 #    flags, but CPU settings - with most of them being flags. Those settings
2912 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2913 #
2914 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2915 # expensive. If you need the value returned from this, you can get it much
2916 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2917 # $accel being 'kvm' or 'tcg'.
2918 #
2919 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2920 # changes, automatically populating pmxcfs.
2921 #
2922 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2923 # since kvm and tcg machines support different flags
2924 #
2925 sub query_supported_cpu_flags {
2926     my ($arch) = @_;
2927
2928     $arch //= get_host_arch();
2929     my $default_machine = $default_machines->{$arch};
2930
2931     my $flags = {};
2932
2933     # FIXME: Once this is merged, the code below should work for ARM as well:
2934     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2935     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2936         $arch eq "aarch64";
2937
2938     my $kvm_supported = defined(kvm_version());
2939     my $qemu_cmd = get_command_for_arch($arch);
2940     my $fakevmid = -1;
2941     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2942
2943     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2944     my $query_supported_run_qemu = sub {
2945         my ($kvm) = @_;
2946
2947         my $flags = {};
2948         my $cmd = [
2949             $qemu_cmd,
2950             '-machine', $default_machine,
2951             '-display', 'none',
2952             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2953             '-mon', 'chardev=qmp,mode=control',
2954             '-pidfile', $pidfile,
2955             '-S', '-daemonize'
2956         ];
2957
2958         if (!$kvm) {
2959             push @$cmd, '-accel', 'tcg';
2960         }
2961
2962         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2963         die "QEMU flag querying VM exited with code " . $rc if $rc;
2964
2965         eval {
2966             my $cmd_result = mon_cmd(
2967                 $fakevmid,
2968                 'query-cpu-model-expansion',
2969                 type => 'full',
2970                 model => { name => 'host' }
2971             );
2972
2973             my $props = $cmd_result->{model}->{props};
2974             foreach my $prop (keys %$props) {
2975                 next if $props->{$prop} ne '1';
2976                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2977                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2978                 # We only keep those with underscores, to match /proc/cpuinfo
2979                 $prop =~ s/\.|-/_/g;
2980                 $flags->{$prop} = 1;
2981             }
2982         };
2983         my $err = $@;
2984
2985         # force stop with 10 sec timeout and 'nocheck'
2986         # always stop, even if QMP failed
2987         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2988
2989         die $err if $err;
2990
2991         return [ sort keys %$flags ];
2992     };
2993
2994     # We need to query QEMU twice, since KVM and TCG have different supported flags
2995     PVE::QemuConfig->lock_config($fakevmid, sub {
2996         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2997         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2998
2999         if ($kvm_supported) {
3000             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3001             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3002         }
3003     });
3004
3005     return $flags;
3006 }
3007
3008 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3009 my $understood_cpu_flag_dir = "/usr/share/kvm";
3010 sub query_understood_cpu_flags {
3011     my $arch = get_host_arch();
3012     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3013
3014     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3015         if ! -e $filepath;
3016
3017     my $raw = file_get_contents($filepath);
3018     $raw =~ s/^\s+|\s+$//g;
3019     my @flags = split(/\s+/, $raw);
3020
3021     return \@flags;
3022 }
3023
3024 sub config_to_command {
3025     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3026
3027     my $cmd = [];
3028     my $globalFlags = [];
3029     my $machineFlags = [];
3030     my $rtcFlags = [];
3031     my $devices = [];
3032     my $pciaddr = '';
3033     my $bridges = {};
3034     my $ostype = $conf->{ostype};
3035     my $winversion = windows_version($ostype);
3036     my $kvm = $conf->{kvm};
3037     my $nodename = nodename();
3038
3039     my $arch = get_vm_arch($conf);
3040     my $kvm_binary = get_command_for_arch($arch);
3041     my $kvmver = kvm_user_version($kvm_binary);
3042
3043     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3044         $kvmver //= "undefined";
3045         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3046     }
3047
3048     my $add_pve_version = min_version($kvmver, 4, 1);
3049
3050     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3051     my $machine_version = extract_version($machine_type, $kvmver);
3052     $kvm //= 1 if is_native($arch);
3053
3054     $machine_version =~ m/(\d+)\.(\d+)/;
3055     my ($machine_major, $machine_minor) = ($1, $2);
3056
3057     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3058         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3059     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3060         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3061             ." please upgrade node '$nodename'\n"
3062     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3063         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3064         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3065             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3066             ." node '$nodename'\n";
3067     }
3068
3069     # if a specific +pve version is required for a feature, use $version_guard
3070     # instead of min_version to allow machines to be run with the minimum
3071     # required version
3072     my $required_pve_version = 0;
3073     my $version_guard = sub {
3074         my ($major, $minor, $pve) = @_;
3075         return 0 if !min_version($machine_version, $major, $minor, $pve);
3076         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3077         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3078         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3079         return 1;
3080     };
3081
3082     if ($kvm && !defined kvm_version()) {
3083         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3084             ." or enable in BIOS.\n";
3085     }
3086
3087     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3088     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3089     my $use_old_bios_files = undef;
3090     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3091
3092     my $cpuunits = defined($conf->{cpuunits}) ?
3093             $conf->{cpuunits} : $defaults->{cpuunits};
3094
3095     push @$cmd, $kvm_binary;
3096
3097     push @$cmd, '-id', $vmid;
3098
3099     my $vmname = $conf->{name} || "vm$vmid";
3100
3101     push @$cmd, '-name', $vmname;
3102
3103     push @$cmd, '-no-shutdown';
3104
3105     my $use_virtio = 0;
3106
3107     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3108     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3109     push @$cmd, '-mon', "chardev=qmp,mode=control";
3110
3111     if (min_version($machine_version, 2, 12)) {
3112         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3113         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3114     }
3115
3116     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3117
3118     push @$cmd, '-daemonize';
3119
3120     if ($conf->{smbios1}) {
3121         my $smbios_conf = parse_smbios1($conf->{smbios1});
3122         if ($smbios_conf->{base64}) {
3123             # Do not pass base64 flag to qemu
3124             delete $smbios_conf->{base64};
3125             my $smbios_string = "";
3126             foreach my $key (keys %$smbios_conf) {
3127                 my $value;
3128                 if ($key eq "uuid") {
3129                     $value = $smbios_conf->{uuid}
3130                 } else {
3131                     $value = decode_base64($smbios_conf->{$key});
3132                 }
3133                 # qemu accepts any binary data, only commas need escaping by double comma
3134                 $value =~ s/,/,,/g;
3135                 $smbios_string .= "," . $key . "=" . $value if $value;
3136             }
3137             push @$cmd, '-smbios', "type=1" . $smbios_string;
3138         } else {
3139             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3140         }
3141     }
3142
3143     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3144         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3145         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3146
3147         my ($path, $format);
3148         if (my $efidisk = $conf->{efidisk0}) {
3149             my $d = parse_drive('efidisk0', $efidisk);
3150             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3151             $format = $d->{format};
3152             if ($storeid) {
3153                 $path = PVE::Storage::path($storecfg, $d->{file});
3154                 if (!defined($format)) {
3155                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3156                     $format = qemu_img_format($scfg, $volname);
3157                 }
3158             } else {
3159                 $path = $d->{file};
3160                 die "efidisk format must be specified\n"
3161                     if !defined($format);
3162             }
3163         } else {
3164             warn "no efidisk configured! Using temporary efivars disk.\n";
3165             $path = "/tmp/$vmid-ovmf.fd";
3166             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3167             $format = 'raw';
3168         }
3169
3170         my $size_str = "";
3171
3172         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3173             $size_str = ",size=" . (-s $ovmf_vars);
3174         }
3175
3176         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3177         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3178     }
3179
3180     # load q35 config
3181     if ($q35) {
3182         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3183         if (min_version($machine_version, 4, 0)) {
3184             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3185         } else {
3186             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3187         }
3188     }
3189
3190     if ($conf->{vmgenid}) {
3191         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3192     }
3193
3194     # add usb controllers
3195     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3196         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3197     push @$devices, @usbcontrollers if @usbcontrollers;
3198     my $vga = parse_vga($conf->{vga});
3199
3200     my $qxlnum = vga_conf_has_spice($conf->{vga});
3201     $vga->{type} = 'qxl' if $qxlnum;
3202
3203     if (!$vga->{type}) {
3204         if ($arch eq 'aarch64') {
3205             $vga->{type} = 'virtio';
3206         } elsif (min_version($machine_version, 2, 9)) {
3207             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3208         } else {
3209             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3210         }
3211     }
3212
3213     # enable absolute mouse coordinates (needed by vnc)
3214     my $tablet;
3215     if (defined($conf->{tablet})) {
3216         $tablet = $conf->{tablet};
3217     } else {
3218         $tablet = $defaults->{tablet};
3219         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3220         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3221     }
3222
3223     if ($tablet) {
3224         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3225         my $kbd = print_keyboarddevice_full($conf, $arch);
3226         push @$devices, '-device', $kbd if defined($kbd);
3227     }
3228
3229     my $bootorder = device_bootorder($conf);
3230
3231     # host pci device passthrough
3232     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3233         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3234
3235     # usb devices
3236     my $usb_dev_features = {};
3237     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3238
3239     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3240         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3241     push @$devices, @usbdevices if @usbdevices;
3242
3243     # serial devices
3244     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3245         if (my $path = $conf->{"serial$i"}) {
3246             if ($path eq 'socket') {
3247                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3248                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3249                 # On aarch64, serial0 is the UART device. Qemu only allows
3250                 # connecting UART devices via the '-serial' command line, as
3251                 # the device has a fixed slot on the hardware...
3252                 if ($arch eq 'aarch64' && $i == 0) {
3253                     push @$devices, '-serial', "chardev:serial$i";
3254                 } else {
3255                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3256                 }
3257             } else {
3258                 die "no such serial device\n" if ! -c $path;
3259                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3260                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3261             }
3262         }
3263     }
3264
3265     # parallel devices
3266     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3267         if (my $path = $conf->{"parallel$i"}) {
3268             die "no such parallel device\n" if ! -c $path;
3269             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3270             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3271             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3272         }
3273     }
3274
3275     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3276         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3277         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3278         push @$devices, @$audio_devs;
3279     }
3280
3281     my $sockets = 1;
3282     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3283     $sockets = $conf->{sockets} if  $conf->{sockets};
3284
3285     my $cores = $conf->{cores} || 1;
3286
3287     my $maxcpus = $sockets * $cores;
3288
3289     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3290
3291     my $allowed_vcpus = $cpuinfo->{cpus};
3292
3293     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3294         if ($allowed_vcpus < $maxcpus);
3295
3296     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3297
3298         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3299         for (my $i = 2; $i <= $vcpus; $i++)  {
3300             my $cpustr = print_cpu_device($conf,$i);
3301             push @$cmd, '-device', $cpustr;
3302         }
3303
3304     } else {
3305
3306         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3307     }
3308     push @$cmd, '-nodefaults';
3309
3310     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3311
3312     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3313
3314     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3315
3316     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3317         push @$devices, '-device', print_vga_device(
3318             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3319         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3320         push @$cmd,  '-vnc', "unix:$socket,password";
3321     } else {
3322         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3323         push @$cmd, '-nographic';
3324     }
3325
3326     # time drift fix
3327     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3328     my $useLocaltime = $conf->{localtime};
3329
3330     if ($winversion >= 5) { # windows
3331         $useLocaltime = 1 if !defined($conf->{localtime});
3332
3333         # use time drift fix when acpi is enabled
3334         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3335             $tdf = 1 if !defined($conf->{tdf});
3336         }
3337     }
3338
3339     if ($winversion >= 6) {
3340         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3341         push @$cmd, '-no-hpet';
3342     }
3343
3344     push @$rtcFlags, 'driftfix=slew' if $tdf;
3345
3346     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3347         push @$rtcFlags, "base=$conf->{startdate}";
3348     } elsif ($useLocaltime) {
3349         push @$rtcFlags, 'base=localtime';
3350     }
3351
3352     if ($forcecpu) {
3353         push @$cmd, '-cpu', $forcecpu;
3354     } else {
3355         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3356     }
3357
3358     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3359
3360     push @$cmd, '-S' if $conf->{freeze};
3361
3362     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3363
3364     my $guest_agent = parse_guest_agent($conf);
3365
3366     if ($guest_agent->{enabled}) {
3367         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3368         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3369
3370         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3371             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3372             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3373             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3374         } elsif ($guest_agent->{type} eq 'isa') {
3375             push @$devices, '-device', "isa-serial,chardev=qga0";
3376         }
3377     }
3378
3379     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3380     if ($rng && $version_guard->(4, 1, 2)) {
3381         check_rng_source($rng->{source});
3382
3383         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3384         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3385         my $limiter_str = "";
3386         if ($max_bytes) {
3387             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3388         }
3389
3390         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3391         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3392         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3393     }
3394
3395     my $spice_port;
3396
3397     if ($qxlnum) {
3398         if ($qxlnum > 1) {
3399             if ($winversion){
3400                 for (my $i = 1; $i < $qxlnum; $i++){
3401                     push @$devices, '-device', print_vga_device(
3402                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3403                 }
3404             } else {
3405                 # assume other OS works like Linux
3406                 my ($ram, $vram) = ("134217728", "67108864");
3407                 if ($vga->{memory}) {
3408                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3409                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3410                 }
3411                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3412                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3413             }
3414         }
3415
3416         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3417
3418         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3419         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3420         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3421
3422         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3423         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3424         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3425
3426         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3427         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3428
3429         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3430         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3431         if ($spice_enhancement->{foldersharing}) {
3432             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3433             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3434         }
3435
3436         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3437         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3438             if $spice_enhancement->{videostreaming};
3439
3440         push @$devices, '-spice', "$spice_opts";
3441     }
3442
3443     # enable balloon by default, unless explicitly disabled
3444     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3445         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3446         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3447     }
3448
3449     if ($conf->{watchdog}) {
3450         my $wdopts = parse_watchdog($conf->{watchdog});
3451         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3452         my $watchdog = $wdopts->{model} || 'i6300esb';
3453         push @$devices, '-device', "$watchdog$pciaddr";
3454         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3455     }
3456
3457     my $vollist = [];
3458     my $scsicontroller = {};
3459     my $ahcicontroller = {};
3460     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3461
3462     # Add iscsi initiator name if available
3463     if (my $initiator = get_initiator_name()) {
3464         push @$devices, '-iscsi', "initiator-name=$initiator";
3465     }
3466
3467     PVE::QemuConfig->foreach_volume($conf, sub {
3468         my ($ds, $drive) = @_;
3469
3470         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3471             push @$vollist, $drive->{file};
3472         }
3473
3474         # ignore efidisk here, already added in bios/fw handling code above
3475         return if $drive->{interface} eq 'efidisk';
3476
3477         $use_virtio = 1 if $ds =~ m/^virtio/;
3478
3479         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3480
3481         if ($drive->{interface} eq 'virtio'){
3482            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3483         }
3484
3485         if ($drive->{interface} eq 'scsi') {
3486
3487             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3488
3489             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3490                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3491
3492             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3493             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3494
3495             my $iothread = '';
3496             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3497                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3498                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3499             } elsif ($drive->{iothread}) {
3500                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3501             }
3502
3503             my $queues = '';
3504             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3505                 $queues = ",num_queues=$drive->{queues}";
3506             }
3507
3508             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3509                 if !$scsicontroller->{$controller};
3510             $scsicontroller->{$controller}=1;
3511         }
3512
3513         if ($drive->{interface} eq 'sata') {
3514             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3515             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3516             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3517                 if !$ahcicontroller->{$controller};
3518             $ahcicontroller->{$controller}=1;
3519         }
3520
3521         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3522         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3523
3524         push @$devices, '-drive',$drive_cmd;
3525         push @$devices, '-device', print_drivedevice_full(
3526             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3527     });
3528
3529     for (my $i = 0; $i < $MAX_NETS; $i++) {
3530         my $netname = "net$i";
3531
3532         next if !$conf->{$netname};
3533         my $d = parse_net($conf->{$netname});
3534         next if !$d;
3535
3536         $use_virtio = 1 if $d->{model} eq 'virtio';
3537
3538         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3539
3540         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3541         push @$devices, '-netdev', $netdevfull;
3542
3543         my $netdevicefull = print_netdevice_full(
3544             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3545
3546         push @$devices, '-device', $netdevicefull;
3547     }
3548
3549     if ($conf->{ivshmem}) {
3550         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3551
3552         my $bus;
3553         if ($q35) {
3554             $bus = print_pcie_addr("ivshmem");
3555         } else {
3556             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3557         }
3558
3559         my $ivshmem_name = $ivshmem->{name} // $vmid;
3560         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3561
3562         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3563         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3564             .",size=$ivshmem->{size}M";
3565     }
3566
3567     # pci.4 is nested in pci.1
3568     $bridges->{1} = 1 if $bridges->{4};
3569
3570     if (!$q35) {
3571         # add pci bridges
3572         if (min_version($machine_version, 2, 3)) {
3573            $bridges->{1} = 1;
3574            $bridges->{2} = 1;
3575         }
3576
3577         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3578
3579     }
3580
3581     for my $k (sort {$b cmp $a} keys %$bridges) {
3582         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3583
3584         my $k_name = $k;
3585         if ($k == 2 && $legacy_igd) {
3586             $k_name = "$k-igd";
3587         }
3588         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3589
3590         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3591         if ($q35) {
3592             # add after -readconfig pve-q35.cfg
3593             splice @$devices, 2, 0, '-device', $devstr;
3594         } else {
3595             unshift @$devices, '-device', $devstr if $k > 0;
3596         }
3597     }
3598
3599     if (!$kvm) {
3600         push @$machineFlags, 'accel=tcg';
3601     }
3602
3603     my $machine_type_min = $machine_type;
3604     if ($add_pve_version) {
3605         $machine_type_min =~ s/\+pve\d+$//;
3606         $machine_type_min .= "+pve$required_pve_version";
3607     }
3608     push @$machineFlags, "type=${machine_type_min}";
3609
3610     push @$cmd, @$devices;
3611     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3612     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3613     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3614
3615     if (my $vmstate = $conf->{vmstate}) {
3616         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3617         push @$vollist, $vmstate;
3618         push @$cmd, '-loadstate', $statepath;
3619         print "activating and using '$vmstate' as vmstate\n";
3620     }
3621
3622     # add custom args
3623     if ($conf->{args}) {
3624         my $aa = PVE::Tools::split_args($conf->{args});
3625         push @$cmd, @$aa;
3626     }
3627
3628     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3629 }
3630
3631 sub check_rng_source {
3632     my ($source) = @_;
3633
3634     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3635     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3636         if ! -e $source;
3637
3638     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3639     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3640         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3641         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3642         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3643             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3644             ." to the host.\n";
3645     }
3646 }
3647
3648 sub spice_port {
3649     my ($vmid) = @_;
3650
3651     my $res = mon_cmd($vmid, 'query-spice');
3652
3653     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3654 }
3655
3656 sub vm_devices_list {
3657     my ($vmid) = @_;
3658
3659     my $res = mon_cmd($vmid, 'query-pci');
3660     my $devices_to_check = [];
3661     my $devices = {};
3662     foreach my $pcibus (@$res) {
3663         push @$devices_to_check, @{$pcibus->{devices}},
3664     }
3665
3666     while (@$devices_to_check) {
3667         my $to_check = [];
3668         for my $d (@$devices_to_check) {
3669             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3670             next if !$d->{'pci_bridge'};
3671
3672             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3673             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3674         }
3675         $devices_to_check = $to_check;
3676     }
3677
3678     my $resblock = mon_cmd($vmid, 'query-block');
3679     foreach my $block (@$resblock) {
3680         if($block->{device} =~ m/^drive-(\S+)/){
3681                 $devices->{$1} = 1;
3682         }
3683     }
3684
3685     my $resmice = mon_cmd($vmid, 'query-mice');
3686     foreach my $mice (@$resmice) {
3687         if ($mice->{name} eq 'QEMU HID Tablet') {
3688             $devices->{tablet} = 1;
3689             last;
3690         }
3691     }
3692
3693     # for usb devices there is no query-usb
3694     # but we can iterate over the entries in
3695     # qom-list path=/machine/peripheral
3696     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3697     foreach my $per (@$resperipheral) {
3698         if ($per->{name} =~ m/^usb\d+$/) {
3699             $devices->{$per->{name}} = 1;
3700         }
3701     }
3702
3703     return $devices;
3704 }
3705
3706 sub vm_deviceplug {
3707     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3708
3709     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3710
3711     my $devices_list = vm_devices_list($vmid);
3712     return 1 if defined($devices_list->{$deviceid});
3713
3714     # add PCI bridge if we need it for the device
3715     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3716
3717     if ($deviceid eq 'tablet') {
3718
3719         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3720
3721     } elsif ($deviceid eq 'keyboard') {
3722
3723         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3724
3725     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3726
3727         die "usb hotplug currently not reliable\n";
3728         # since we can't reliably hot unplug all added usb devices and usb
3729         # passthrough breaks live migration we disable usb hotplugging for now
3730         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3731
3732     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3733
3734         qemu_iothread_add($vmid, $deviceid, $device);
3735
3736         qemu_driveadd($storecfg, $vmid, $device);
3737         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3738
3739         qemu_deviceadd($vmid, $devicefull);
3740         eval { qemu_deviceaddverify($vmid, $deviceid); };
3741         if (my $err = $@) {
3742             eval { qemu_drivedel($vmid, $deviceid); };
3743             warn $@ if $@;
3744             die $err;
3745         }
3746
3747     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3748
3749
3750         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3751         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3752         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3753
3754         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3755
3756         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3757             qemu_iothread_add($vmid, $deviceid, $device);
3758             $devicefull .= ",iothread=iothread-$deviceid";
3759         }
3760
3761         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3762             $devicefull .= ",num_queues=$device->{queues}";
3763         }
3764
3765         qemu_deviceadd($vmid, $devicefull);
3766         qemu_deviceaddverify($vmid, $deviceid);
3767
3768     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3769
3770         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3771         qemu_driveadd($storecfg, $vmid, $device);
3772
3773         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3774         eval { qemu_deviceadd($vmid, $devicefull); };
3775         if (my $err = $@) {
3776             eval { qemu_drivedel($vmid, $deviceid); };
3777             warn $@ if $@;
3778             die $err;
3779         }
3780
3781     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3782
3783         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3784
3785         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3786         my $use_old_bios_files = undef;
3787         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3788
3789         my $netdevicefull = print_netdevice_full(
3790             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3791         qemu_deviceadd($vmid, $netdevicefull);
3792         eval {
3793             qemu_deviceaddverify($vmid, $deviceid);
3794             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3795         };
3796         if (my $err = $@) {
3797             eval { qemu_netdevdel($vmid, $deviceid); };
3798             warn $@ if $@;
3799             die $err;
3800         }
3801
3802     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3803
3804         my $bridgeid = $2;
3805         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3806         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3807
3808         qemu_deviceadd($vmid, $devicefull);
3809         qemu_deviceaddverify($vmid, $deviceid);
3810
3811     } else {
3812         die "can't hotplug device '$deviceid'\n";
3813     }
3814
3815     return 1;
3816 }
3817
3818 # fixme: this should raise exceptions on error!
3819 sub vm_deviceunplug {
3820     my ($vmid, $conf, $deviceid) = @_;
3821
3822     my $devices_list = vm_devices_list($vmid);
3823     return 1 if !defined($devices_list->{$deviceid});
3824
3825     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3826     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3827
3828     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3829
3830         qemu_devicedel($vmid, $deviceid);
3831
3832     } elsif ($deviceid =~ m/^usb\d+$/) {
3833
3834         die "usb hotplug currently not reliable\n";
3835         # when unplugging usb devices this way, there may be remaining usb
3836         # controllers/hubs so we disable it for now
3837         #qemu_devicedel($vmid, $deviceid);
3838         #qemu_devicedelverify($vmid, $deviceid);
3839
3840     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3841
3842         qemu_devicedel($vmid, $deviceid);
3843         qemu_devicedelverify($vmid, $deviceid);
3844         qemu_drivedel($vmid, $deviceid);
3845         qemu_iothread_del($conf, $vmid, $deviceid);
3846
3847     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3848
3849         qemu_devicedel($vmid, $deviceid);
3850         qemu_devicedelverify($vmid, $deviceid);
3851         qemu_iothread_del($conf, $vmid, $deviceid);
3852
3853     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3854
3855         qemu_devicedel($vmid, $deviceid);
3856         qemu_drivedel($vmid, $deviceid);
3857         qemu_deletescsihw($conf, $vmid, $deviceid);
3858
3859     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3860
3861         qemu_devicedel($vmid, $deviceid);
3862         qemu_devicedelverify($vmid, $deviceid);
3863         qemu_netdevdel($vmid, $deviceid);
3864
3865     } else {
3866         die "can't unplug device '$deviceid'\n";
3867     }
3868
3869     return 1;
3870 }
3871
3872 sub qemu_deviceadd {
3873     my ($vmid, $devicefull) = @_;
3874
3875     $devicefull = "driver=".$devicefull;
3876     my %options =  split(/[=,]/, $devicefull);
3877
3878     mon_cmd($vmid, "device_add" , %options);
3879 }
3880
3881 sub qemu_devicedel {
3882     my ($vmid, $deviceid) = @_;
3883
3884     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3885 }
3886
3887 sub qemu_iothread_add {
3888     my($vmid, $deviceid, $device) = @_;
3889
3890     if ($device->{iothread}) {
3891         my $iothreads = vm_iothreads_list($vmid);
3892         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3893     }
3894 }
3895
3896 sub qemu_iothread_del {
3897     my($conf, $vmid, $deviceid) = @_;
3898
3899     my $confid = $deviceid;
3900     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3901         $confid = 'scsi' . $1;
3902     }
3903     my $device = parse_drive($confid, $conf->{$confid});
3904     if ($device->{iothread}) {
3905         my $iothreads = vm_iothreads_list($vmid);
3906         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3907     }
3908 }
3909
3910 sub qemu_objectadd {
3911     my($vmid, $objectid, $qomtype) = @_;
3912
3913     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3914
3915     return 1;
3916 }
3917
3918 sub qemu_objectdel {
3919     my($vmid, $objectid) = @_;
3920
3921     mon_cmd($vmid, "object-del", id => $objectid);
3922
3923     return 1;
3924 }
3925
3926 sub qemu_driveadd {
3927     my ($storecfg, $vmid, $device) = @_;
3928
3929     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3930     $drive =~ s/\\/\\\\/g;
3931     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3932
3933     # If the command succeeds qemu prints: "OK"
3934     return 1 if $ret =~ m/OK/s;
3935
3936     die "adding drive failed: $ret\n";
3937 }
3938
3939 sub qemu_drivedel {
3940     my($vmid, $deviceid) = @_;
3941
3942     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3943     $ret =~ s/^\s+//;
3944
3945     return 1 if $ret eq "";
3946
3947     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3948     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3949
3950     die "deleting drive $deviceid failed : $ret\n";
3951 }
3952
3953 sub qemu_deviceaddverify {
3954     my ($vmid, $deviceid) = @_;
3955
3956     for (my $i = 0; $i <= 5; $i++) {
3957          my $devices_list = vm_devices_list($vmid);
3958          return 1 if defined($devices_list->{$deviceid});
3959          sleep 1;
3960     }
3961
3962     die "error on hotplug device '$deviceid'\n";
3963 }
3964
3965
3966 sub qemu_devicedelverify {
3967     my ($vmid, $deviceid) = @_;
3968
3969     # need to verify that the device is correctly removed as device_del
3970     # is async and empty return is not reliable
3971
3972     for (my $i = 0; $i <= 5; $i++) {
3973          my $devices_list = vm_devices_list($vmid);
3974          return 1 if !defined($devices_list->{$deviceid});
3975          sleep 1;
3976     }
3977
3978     die "error on hot-unplugging device '$deviceid'\n";
3979 }
3980
3981 sub qemu_findorcreatescsihw {
3982     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3983
3984     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3985
3986     my $scsihwid="$controller_prefix$controller";
3987     my $devices_list = vm_devices_list($vmid);
3988
3989     if(!defined($devices_list->{$scsihwid})) {
3990         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3991     }
3992
3993     return 1;
3994 }
3995
3996 sub qemu_deletescsihw {
3997     my ($conf, $vmid, $opt) = @_;
3998
3999     my $device = parse_drive($opt, $conf->{$opt});
4000
4001     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4002         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4003         return 1;
4004     }
4005
4006     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4007
4008     my $devices_list = vm_devices_list($vmid);
4009     foreach my $opt (keys %{$devices_list}) {
4010         if (is_valid_drivename($opt)) {
4011             my $drive = parse_drive($opt, $conf->{$opt});
4012             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4013                 return 1;
4014             }
4015         }
4016     }
4017
4018     my $scsihwid="scsihw$controller";
4019
4020     vm_deviceunplug($vmid, $conf, $scsihwid);
4021
4022     return 1;
4023 }
4024
4025 sub qemu_add_pci_bridge {
4026     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4027
4028     my $bridges = {};
4029
4030     my $bridgeid;
4031
4032     print_pci_addr($device, $bridges, $arch, $machine_type);
4033
4034     while (my ($k, $v) = each %$bridges) {
4035         $bridgeid = $k;
4036     }
4037     return 1 if !defined($bridgeid) || $bridgeid < 1;
4038
4039     my $bridge = "pci.$bridgeid";
4040     my $devices_list = vm_devices_list($vmid);
4041
4042     if (!defined($devices_list->{$bridge})) {
4043         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4044     }
4045
4046     return 1;
4047 }
4048
4049 sub qemu_set_link_status {
4050     my ($vmid, $device, $up) = @_;
4051
4052     mon_cmd($vmid, "set_link", name => $device,
4053                up => $up ? JSON::true : JSON::false);
4054 }
4055
4056 sub qemu_netdevadd {
4057     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4058
4059     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4060     my %options =  split(/[=,]/, $netdev);
4061
4062     if (defined(my $vhost = $options{vhost})) {
4063         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4064     }
4065
4066     if (defined(my $queues = $options{queues})) {
4067         $options{queues} = $queues + 0;
4068     }
4069
4070     mon_cmd($vmid, "netdev_add",  %options);
4071     return 1;
4072 }
4073
4074 sub qemu_netdevdel {
4075     my ($vmid, $deviceid) = @_;
4076
4077     mon_cmd($vmid, "netdev_del", id => $deviceid);
4078 }
4079
4080 sub qemu_usb_hotplug {
4081     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4082
4083     return if !$device;
4084
4085     # remove the old one first
4086     vm_deviceunplug($vmid, $conf, $deviceid);
4087
4088     # check if xhci controller is necessary and available
4089     if ($device->{usb3}) {
4090
4091         my $devicelist = vm_devices_list($vmid);
4092
4093         if (!$devicelist->{xhci}) {
4094             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4095             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4096         }
4097     }
4098     my $d = parse_usb_device($device->{host});
4099     $d->{usb3} = $device->{usb3};
4100
4101     # add the new one
4102     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4103 }
4104
4105 sub qemu_cpu_hotplug {
4106     my ($vmid, $conf, $vcpus) = @_;
4107
4108     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4109
4110     my $sockets = 1;
4111     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4112     $sockets = $conf->{sockets} if  $conf->{sockets};
4113     my $cores = $conf->{cores} || 1;
4114     my $maxcpus = $sockets * $cores;
4115
4116     $vcpus = $maxcpus if !$vcpus;
4117
4118     die "you can't add more vcpus than maxcpus\n"
4119         if $vcpus > $maxcpus;
4120
4121     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4122
4123     if ($vcpus < $currentvcpus) {
4124
4125         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4126
4127             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4128                 qemu_devicedel($vmid, "cpu$i");
4129                 my $retry = 0;
4130                 my $currentrunningvcpus = undef;
4131                 while (1) {
4132                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4133                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4134                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4135                     $retry++;
4136                     sleep 1;
4137                 }
4138                 #update conf after each succesfull cpu unplug
4139                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4140                 PVE::QemuConfig->write_config($vmid, $conf);
4141             }
4142         } else {
4143             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4144         }
4145
4146         return;
4147     }
4148
4149     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4150     die "vcpus in running vm does not match its configuration\n"
4151         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4152
4153     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4154
4155         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4156             my $cpustr = print_cpu_device($conf, $i);
4157             qemu_deviceadd($vmid, $cpustr);
4158
4159             my $retry = 0;
4160             my $currentrunningvcpus = undef;
4161             while (1) {
4162                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4163                 last if scalar(@{$currentrunningvcpus}) == $i;
4164                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4165                 sleep 1;
4166                 $retry++;
4167             }
4168             #update conf after each succesfull cpu hotplug
4169             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4170             PVE::QemuConfig->write_config($vmid, $conf);
4171         }
4172     } else {
4173
4174         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4175             mon_cmd($vmid, "cpu-add", id => int($i));
4176         }
4177     }
4178 }
4179
4180 sub qemu_block_set_io_throttle {
4181     my ($vmid, $deviceid,
4182         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4183         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4184         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4185         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4186
4187     return if !check_running($vmid) ;
4188
4189     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4190         bps => int($bps),
4191         bps_rd => int($bps_rd),
4192         bps_wr => int($bps_wr),
4193         iops => int($iops),
4194         iops_rd => int($iops_rd),
4195         iops_wr => int($iops_wr),
4196         bps_max => int($bps_max),
4197         bps_rd_max => int($bps_rd_max),
4198         bps_wr_max => int($bps_wr_max),
4199         iops_max => int($iops_max),
4200         iops_rd_max => int($iops_rd_max),
4201         iops_wr_max => int($iops_wr_max),
4202         bps_max_length => int($bps_max_length),
4203         bps_rd_max_length => int($bps_rd_max_length),
4204         bps_wr_max_length => int($bps_wr_max_length),
4205         iops_max_length => int($iops_max_length),
4206         iops_rd_max_length => int($iops_rd_max_length),
4207         iops_wr_max_length => int($iops_wr_max_length),
4208     );
4209
4210 }
4211
4212 sub qemu_block_resize {
4213     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4214
4215     my $running = check_running($vmid);
4216
4217     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4218
4219     return if !$running;
4220
4221     my $padding = (1024 - $size % 1024) % 1024;
4222     $size = $size + $padding;
4223
4224     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4225
4226 }
4227
4228 sub qemu_volume_snapshot {
4229     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4230
4231     my $running = check_running($vmid);
4232
4233     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4234         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4235     } else {
4236         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4237     }
4238 }
4239
4240 sub qemu_volume_snapshot_delete {
4241     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4242
4243     my $running = check_running($vmid);
4244
4245     if($running) {
4246
4247         $running = undef;
4248         my $conf = PVE::QemuConfig->load_config($vmid);
4249         PVE::QemuConfig->foreach_volume($conf, sub {
4250             my ($ds, $drive) = @_;
4251             $running = 1 if $drive->{file} eq $volid;
4252         });
4253     }
4254
4255     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4256         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4257     } else {
4258         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4259     }
4260 }
4261
4262 sub set_migration_caps {
4263     my ($vmid) = @_;
4264
4265     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4266
4267     my $cap_ref = [];
4268
4269     my $enabled_cap = {
4270         "auto-converge" => 1,
4271         "xbzrle" => 1,
4272         "x-rdma-pin-all" => 0,
4273         "zero-blocks" => 0,
4274         "compress" => 0,
4275         "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0,
4276     };
4277
4278     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4279
4280     for my $supported_capability (@$supported_capabilities) {
4281         push @$cap_ref, {
4282             capability => $supported_capability->{capability},
4283             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4284         };
4285     }
4286
4287     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4288 }
4289
4290 sub foreach_volid {
4291     my ($conf, $func, @param) = @_;
4292
4293     my $volhash = {};
4294
4295     my $test_volid = sub {
4296         my ($key, $drive, $snapname) = @_;
4297
4298         my $volid = $drive->{file};
4299         return if !$volid;
4300
4301         $volhash->{$volid}->{cdrom} //= 1;
4302         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4303
4304         my $replicate = $drive->{replicate} // 1;
4305         $volhash->{$volid}->{replicate} //= 0;
4306         $volhash->{$volid}->{replicate} = 1 if $replicate;
4307
4308         $volhash->{$volid}->{shared} //= 0;
4309         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4310
4311         $volhash->{$volid}->{referenced_in_config} //= 0;
4312         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4313
4314         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4315             if defined($snapname);
4316
4317         my $size = $drive->{size};
4318         $volhash->{$volid}->{size} //= $size if $size;
4319
4320         $volhash->{$volid}->{is_vmstate} //= 0;
4321         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4322
4323         $volhash->{$volid}->{is_unused} //= 0;
4324         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4325     };
4326
4327     my $include_opts = {
4328         extra_keys => ['vmstate'],
4329         include_unused => 1,
4330     };
4331
4332     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4333     foreach my $snapname (keys %{$conf->{snapshots}}) {
4334         my $snap = $conf->{snapshots}->{$snapname};
4335         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4336     }
4337
4338     foreach my $volid (keys %$volhash) {
4339         &$func($volid, $volhash->{$volid}, @param);
4340     }
4341 }
4342
4343 my $fast_plug_option = {
4344     'lock' => 1,
4345     'name' => 1,
4346     'onboot' => 1,
4347     'shares' => 1,
4348     'startup' => 1,
4349     'description' => 1,
4350     'protection' => 1,
4351     'vmstatestorage' => 1,
4352     'hookscript' => 1,
4353     'tags' => 1,
4354 };
4355
4356 # hotplug changes in [PENDING]
4357 # $selection hash can be used to only apply specified options, for
4358 # example: { cores => 1 } (only apply changed 'cores')
4359 # $errors ref is used to return error messages
4360 sub vmconfig_hotplug_pending {
4361     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4362
4363     my $defaults = load_defaults();
4364     my $arch = get_vm_arch($conf);
4365     my $machine_type = get_vm_machine($conf, undef, $arch);
4366
4367     # commit values which do not have any impact on running VM first
4368     # Note: those option cannot raise errors, we we do not care about
4369     # $selection and always apply them.
4370
4371     my $add_error = sub {
4372         my ($opt, $msg) = @_;
4373         $errors->{$opt} = "hotplug problem - $msg";
4374     };
4375
4376     my $changes = 0;
4377     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4378         if ($fast_plug_option->{$opt}) {
4379             $conf->{$opt} = $conf->{pending}->{$opt};
4380             delete $conf->{pending}->{$opt};
4381             $changes = 1;
4382         }
4383     }
4384
4385     if ($changes) {
4386         PVE::QemuConfig->write_config($vmid, $conf);
4387     }
4388
4389     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4390
4391     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4392     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4393     foreach my $opt (sort keys %$pending_delete_hash) {
4394         next if $selection && !$selection->{$opt};
4395         my $force = $pending_delete_hash->{$opt}->{force};
4396         eval {
4397             if ($opt eq 'hotplug') {
4398                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4399             } elsif ($opt eq 'tablet') {
4400                 die "skip\n" if !$hotplug_features->{usb};
4401                 if ($defaults->{tablet}) {
4402                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4403                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4404                         if $arch eq 'aarch64';
4405                 } else {
4406                     vm_deviceunplug($vmid, $conf, 'tablet');
4407                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4408                 }
4409             } elsif ($opt =~ m/^usb\d+/) {
4410                 die "skip\n";
4411                 # since we cannot reliably hot unplug usb devices we are disabling it
4412                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4413                 #vm_deviceunplug($vmid, $conf, $opt);
4414             } elsif ($opt eq 'vcpus') {
4415                 die "skip\n" if !$hotplug_features->{cpu};
4416                 qemu_cpu_hotplug($vmid, $conf, undef);
4417             } elsif ($opt eq 'balloon') {
4418                 # enable balloon device is not hotpluggable
4419                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4420                 # here we reset the ballooning value to memory
4421                 my $balloon = $conf->{memory} || $defaults->{memory};
4422                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4423             } elsif ($fast_plug_option->{$opt}) {
4424                 # do nothing
4425             } elsif ($opt =~ m/^net(\d+)$/) {
4426                 die "skip\n" if !$hotplug_features->{network};
4427                 vm_deviceunplug($vmid, $conf, $opt);
4428             } elsif (is_valid_drivename($opt)) {
4429                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4430                 vm_deviceunplug($vmid, $conf, $opt);
4431                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4432             } elsif ($opt =~ m/^memory$/) {
4433                 die "skip\n" if !$hotplug_features->{memory};
4434                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4435             } elsif ($opt eq 'cpuunits') {
4436                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4437             } elsif ($opt eq 'cpulimit') {
4438                 $cgroup->change_cpu_quota(-1, 100000);
4439             } else {
4440                 die "skip\n";
4441             }
4442         };
4443         if (my $err = $@) {
4444             &$add_error($opt, $err) if $err ne "skip\n";
4445         } else {
4446             delete $conf->{$opt};
4447             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4448         }
4449     }
4450
4451     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4452     $apply_pending_cloudinit = sub {
4453         return if $apply_pending_cloudinit_done; # once is enough
4454         $apply_pending_cloudinit_done = 1; # once is enough
4455
4456         my ($key, $value) = @_;
4457
4458         my @cloudinit_opts = keys %$confdesc_cloudinit;
4459         foreach my $opt (keys %{$conf->{pending}}) {
4460             next if !grep { $_ eq $opt } @cloudinit_opts;
4461             $conf->{$opt} = delete $conf->{pending}->{$opt};
4462         }
4463
4464         my $new_conf = { %$conf };
4465         $new_conf->{$key} = $value;
4466         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4467     };
4468
4469     foreach my $opt (keys %{$conf->{pending}}) {
4470         next if $selection && !$selection->{$opt};
4471         my $value = $conf->{pending}->{$opt};
4472         eval {
4473             if ($opt eq 'hotplug') {
4474                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4475             } elsif ($opt eq 'tablet') {
4476                 die "skip\n" if !$hotplug_features->{usb};
4477                 if ($value == 1) {
4478                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4479                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4480                         if $arch eq 'aarch64';
4481                 } elsif ($value == 0) {
4482                     vm_deviceunplug($vmid, $conf, 'tablet');
4483                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4484                 }
4485             } elsif ($opt =~ m/^usb\d+$/) {
4486                 die "skip\n";
4487                 # since we cannot reliably hot unplug usb devices we disable it for now
4488                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4489                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4490                 #die "skip\n" if !$d;
4491                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4492             } elsif ($opt eq 'vcpus') {
4493                 die "skip\n" if !$hotplug_features->{cpu};
4494                 qemu_cpu_hotplug($vmid, $conf, $value);
4495             } elsif ($opt eq 'balloon') {
4496                 # enable/disable balloning device is not hotpluggable
4497                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4498                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4499                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4500
4501                 # allow manual ballooning if shares is set to zero
4502                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4503                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4504                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4505                 }
4506             } elsif ($opt =~ m/^net(\d+)$/) {
4507                 # some changes can be done without hotplug
4508                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4509                                     $vmid, $opt, $value, $arch, $machine_type);
4510             } elsif (is_valid_drivename($opt)) {
4511                 die "skip\n" if $opt eq 'efidisk0';
4512                 # some changes can be done without hotplug
4513                 my $drive = parse_drive($opt, $value);
4514                 if (drive_is_cloudinit($drive)) {
4515                     &$apply_pending_cloudinit($opt, $value);
4516                 }
4517                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4518                                      $vmid, $opt, $value, $arch, $machine_type);
4519             } elsif ($opt =~ m/^memory$/) { #dimms
4520                 die "skip\n" if !$hotplug_features->{memory};
4521                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4522             } elsif ($opt eq 'cpuunits') {
4523                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4524             } elsif ($opt eq 'cpulimit') {
4525                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4526                 $cgroup->change_cpu_quota($cpulimit, 100000);
4527             } else {
4528                 die "skip\n";  # skip non-hot-pluggable options
4529             }
4530         };
4531         if (my $err = $@) {
4532             &$add_error($opt, $err) if $err ne "skip\n";
4533         } else {
4534             $conf->{$opt} = $value;
4535             delete $conf->{pending}->{$opt};
4536         }
4537     }
4538
4539     PVE::QemuConfig->write_config($vmid, $conf);
4540 }
4541
4542 sub try_deallocate_drive {
4543     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4544
4545     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4546         my $volid = $drive->{file};
4547         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4548             my $sid = PVE::Storage::parse_volume_id($volid);
4549             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4550
4551             # check if the disk is really unused
4552             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4553                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4554             PVE::Storage::vdisk_free($storecfg, $volid);
4555             return 1;
4556         } else {
4557             # If vm is not owner of this disk remove from config
4558             return 1;
4559         }
4560     }
4561
4562     return;
4563 }
4564
4565 sub vmconfig_delete_or_detach_drive {
4566     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4567
4568     my $drive = parse_drive($opt, $conf->{$opt});
4569
4570     my $rpcenv = PVE::RPCEnvironment::get();
4571     my $authuser = $rpcenv->get_user();
4572
4573     if ($force) {
4574         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4575         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4576     } else {
4577         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4578     }
4579 }
4580
4581
4582
4583 sub vmconfig_apply_pending {
4584     my ($vmid, $conf, $storecfg, $errors) = @_;
4585
4586     my $add_apply_error = sub {
4587         my ($opt, $msg) = @_;
4588         my $err_msg = "unable to apply pending change $opt : $msg";
4589         $errors->{$opt} = $err_msg;
4590         warn $err_msg;
4591     };
4592
4593     # cold plug
4594
4595     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4596     foreach my $opt (sort keys %$pending_delete_hash) {
4597         my $force = $pending_delete_hash->{$opt}->{force};
4598         eval {
4599             if ($opt =~ m/^unused/) {
4600                 die "internal error";
4601             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4602                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4603             }
4604         };
4605         if (my $err = $@) {
4606             $add_apply_error->($opt, $err);
4607         } else {
4608             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4609             delete $conf->{$opt};
4610         }
4611     }
4612
4613     PVE::QemuConfig->cleanup_pending($conf);
4614
4615     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4616         next if $opt eq 'delete'; # just to be sure
4617         eval {
4618             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4619                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4620             }
4621         };
4622         if (my $err = $@) {
4623             $add_apply_error->($opt, $err);
4624         } else {
4625             $conf->{$opt} = delete $conf->{pending}->{$opt};
4626         }
4627     }
4628
4629     # write all changes at once to avoid unnecessary i/o
4630     PVE::QemuConfig->write_config($vmid, $conf);
4631 }
4632
4633 sub vmconfig_update_net {
4634     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4635
4636     my $newnet = parse_net($value);
4637
4638     if ($conf->{$opt}) {
4639         my $oldnet = parse_net($conf->{$opt});
4640
4641         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4642             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4643             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4644             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4645
4646             # for non online change, we try to hot-unplug
4647             die "skip\n" if !$hotplug;
4648             vm_deviceunplug($vmid, $conf, $opt);
4649         } else {
4650
4651             die "internal error" if $opt !~ m/net(\d+)/;
4652             my $iface = "tap${vmid}i$1";
4653
4654             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4655                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4656                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4657                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4658                 PVE::Network::tap_unplug($iface);
4659
4660                 if ($have_sdn) {
4661                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4662                 } else {
4663                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4664                 }
4665             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4666                 # Rate can be applied on its own but any change above needs to
4667                 # include the rate in tap_plug since OVS resets everything.
4668                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4669             }
4670
4671             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4672                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4673             }
4674
4675             return 1;
4676         }
4677     }
4678
4679     if ($hotplug) {
4680         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4681     } else {
4682         die "skip\n";
4683     }
4684 }
4685
4686 sub vmconfig_update_disk {
4687     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4688
4689     my $drive = parse_drive($opt, $value);
4690
4691     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4692         my $media = $drive->{media} || 'disk';
4693         my $oldmedia = $old_drive->{media} || 'disk';
4694         die "unable to change media type\n" if $media ne $oldmedia;
4695
4696         if (!drive_is_cdrom($old_drive)) {
4697
4698             if ($drive->{file} ne $old_drive->{file}) {
4699
4700                 die "skip\n" if !$hotplug;
4701
4702                 # unplug and register as unused
4703                 vm_deviceunplug($vmid, $conf, $opt);
4704                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4705
4706             } else {
4707                 # update existing disk
4708
4709                 # skip non hotpluggable value
4710                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4711                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4712                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4713                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4714                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4715                     die "skip\n";
4716                 }
4717
4718                 # apply throttle
4719                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4720                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4721                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4722                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4723                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4724                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4725                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4726                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4727                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4728                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4729                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4730                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4731                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4732                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4733                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4734                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4735                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4736                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4737
4738                     qemu_block_set_io_throttle(
4739                         $vmid,"drive-$opt",
4740                         ($drive->{mbps} || 0)*1024*1024,
4741                         ($drive->{mbps_rd} || 0)*1024*1024,
4742                         ($drive->{mbps_wr} || 0)*1024*1024,
4743                          $drive->{iops} || 0,
4744                          $drive->{iops_rd} || 0,
4745                          $drive->{iops_wr} || 0,
4746                         ($drive->{mbps_max} || 0)*1024*1024,
4747                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4748                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4749                          $drive->{iops_max} || 0,
4750                          $drive->{iops_rd_max} || 0,
4751                          $drive->{iops_wr_max} || 0,
4752                          $drive->{bps_max_length} || 1,
4753                          $drive->{bps_rd_max_length} || 1,
4754                          $drive->{bps_wr_max_length} || 1,
4755                          $drive->{iops_max_length} || 1,
4756                          $drive->{iops_rd_max_length} || 1,
4757                          $drive->{iops_wr_max_length} || 1,
4758                     );
4759
4760                 }
4761
4762                 return 1;
4763             }
4764
4765         } else { # cdrom
4766
4767             if ($drive->{file} eq 'none') {
4768                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4769                 if (drive_is_cloudinit($old_drive)) {
4770                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4771                 }
4772             } else {
4773                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4774
4775                 # force eject if locked
4776                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4777
4778                 if ($path) {
4779                     mon_cmd($vmid, "blockdev-change-medium",
4780                         id => "$opt", filename => "$path");
4781                 }
4782             }
4783
4784             return 1;
4785         }
4786     }
4787
4788     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4789     # hotplug new disks
4790     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4791     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4792 }
4793
4794 # called in locked context by incoming migration
4795 sub vm_migrate_get_nbd_disks {
4796     my ($storecfg, $conf, $replicated_volumes) = @_;
4797
4798     my $local_volumes = {};
4799     PVE::QemuConfig->foreach_volume($conf, sub {
4800         my ($ds, $drive) = @_;
4801
4802         return if drive_is_cdrom($drive);
4803
4804         my $volid = $drive->{file};
4805
4806         return if !$volid;
4807
4808         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4809
4810         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4811         return if $scfg->{shared};
4812
4813         # replicated disks re-use existing state via bitmap
4814         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4815         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4816     });
4817     return $local_volumes;
4818 }
4819
4820 # called in locked context by incoming migration
4821 sub vm_migrate_alloc_nbd_disks {
4822     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4823
4824     my $format = undef;
4825
4826     my $nbd = {};
4827     foreach my $opt (sort keys %$source_volumes) {
4828         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4829
4830         if ($use_existing) {
4831             $nbd->{$opt}->{drivestr} = print_drive($drive);
4832             $nbd->{$opt}->{volid} = $volid;
4833             $nbd->{$opt}->{replicated} = 1;
4834             next;
4835         }
4836
4837         # If a remote storage is specified and the format of the original
4838         # volume is not available there, fall back to the default format.
4839         # Otherwise use the same format as the original.
4840         if (!$storagemap->{identity}) {
4841             $storeid = map_storage($storagemap, $storeid);
4842             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4843             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4844             my $fileFormat = qemu_img_format($scfg, $volname);
4845             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4846         } else {
4847             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4848             $format = qemu_img_format($scfg, $volname);
4849         }
4850
4851         my $size = $drive->{size} / 1024;
4852         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4853         my $newdrive = $drive;
4854         $newdrive->{format} = $format;
4855         $newdrive->{file} = $newvolid;
4856         my $drivestr = print_drive($newdrive);
4857         $nbd->{$opt}->{drivestr} = $drivestr;
4858         $nbd->{$opt}->{volid} = $newvolid;
4859     }
4860
4861     return $nbd;
4862 }
4863
4864 # see vm_start_nolock for parameters, additionally:
4865 # migrate_opts:
4866 #   storagemap = parsed storage map for allocating NBD disks
4867 sub vm_start {
4868     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4869
4870     return PVE::QemuConfig->lock_config($vmid, sub {
4871         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4872
4873         die "you can't start a vm if it's a template\n"
4874             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4875
4876         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4877         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4878
4879         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4880
4881         if ($has_backup_lock && $running) {
4882             # a backup is currently running, attempt to start the guest in the
4883             # existing QEMU instance
4884             return vm_resume($vmid);
4885         }
4886
4887         PVE::QemuConfig->check_lock($conf)
4888             if !($params->{skiplock} || $has_suspended_lock);
4889
4890         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4891
4892         die "VM $vmid already running\n" if $running;
4893
4894         if (my $storagemap = $migrate_opts->{storagemap}) {
4895             my $replicated = $migrate_opts->{replicated_volumes};
4896             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4897             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4898
4899             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4900                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4901             }
4902         }
4903
4904         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4905     });
4906 }
4907
4908
4909 # params:
4910 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4911 #   skiplock => 0/1, skip checking for config lock
4912 #   skiptemplate => 0/1, skip checking whether VM is template
4913 #   forcemachine => to force Qemu machine (rollback/migration)
4914 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4915 #   timeout => in seconds
4916 #   paused => start VM in paused state (backup)
4917 #   resume => resume from hibernation
4918 # migrate_opts:
4919 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4920 #   migratedfrom => source node
4921 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4922 #   network => CIDR of migration network
4923 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4924 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4925 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4926 sub vm_start_nolock {
4927     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4928
4929     my $statefile = $params->{statefile};
4930     my $resume = $params->{resume};
4931
4932     my $migratedfrom = $migrate_opts->{migratedfrom};
4933     my $migration_type = $migrate_opts->{type};
4934
4935     my $res = {};
4936
4937     # clean up leftover reboot request files
4938     eval { clear_reboot_request($vmid); };
4939     warn $@ if $@;
4940
4941     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4942         vmconfig_apply_pending($vmid, $conf, $storecfg);
4943         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4944     }
4945
4946     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4947
4948     my $defaults = load_defaults();
4949
4950     # set environment variable useful inside network script
4951     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4952
4953     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4954
4955     my $forcemachine = $params->{forcemachine};
4956     my $forcecpu = $params->{forcecpu};
4957     if ($resume) {
4958         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4959         $forcemachine = $conf->{runningmachine};
4960         $forcecpu = $conf->{runningcpu};
4961         print "Resuming suspended VM\n";
4962     }
4963
4964     my ($cmd, $vollist, $spice_port) =
4965         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4966
4967     my $migration_ip;
4968     my $get_migration_ip = sub {
4969         my ($nodename) = @_;
4970
4971         return $migration_ip if defined($migration_ip);
4972
4973         my $cidr = $migrate_opts->{network};
4974
4975         if (!defined($cidr)) {
4976             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4977             $cidr = $dc_conf->{migration}->{network};
4978         }
4979
4980         if (defined($cidr)) {
4981             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4982
4983             die "could not get IP: no address configured on local " .
4984                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4985
4986             die "could not get IP: multiple addresses configured on local " .
4987                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4988
4989             $migration_ip = @$ips[0];
4990         }
4991
4992         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4993             if !defined($migration_ip);
4994
4995         return $migration_ip;
4996     };
4997
4998     my $migrate_uri;
4999     if ($statefile) {
5000         if ($statefile eq 'tcp') {
5001             my $localip = "localhost";
5002             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5003             my $nodename = nodename();
5004
5005             if (!defined($migration_type)) {
5006                 if (defined($datacenterconf->{migration}->{type})) {
5007                     $migration_type = $datacenterconf->{migration}->{type};
5008                 } else {
5009                     $migration_type = 'secure';
5010                 }
5011             }
5012
5013             if ($migration_type eq 'insecure') {
5014                 $localip = $get_migration_ip->($nodename);
5015                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5016             }
5017
5018             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5019             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5020             $migrate_uri = "tcp:${localip}:${migrate_port}";
5021             push @$cmd, '-incoming', $migrate_uri;
5022             push @$cmd, '-S';
5023
5024         } elsif ($statefile eq 'unix') {
5025             # should be default for secure migrations as a ssh TCP forward
5026             # tunnel is not deterministic reliable ready and fails regurarly
5027             # to set up in time, so use UNIX socket forwards
5028             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5029             unlink $socket_addr;
5030
5031             $migrate_uri = "unix:$socket_addr";
5032
5033             push @$cmd, '-incoming', $migrate_uri;
5034             push @$cmd, '-S';
5035
5036         } elsif (-e $statefile) {
5037             push @$cmd, '-loadstate', $statefile;
5038         } else {
5039             my $statepath = PVE::Storage::path($storecfg, $statefile);
5040             push @$vollist, $statefile;
5041             push @$cmd, '-loadstate', $statepath;
5042         }
5043     } elsif ($params->{paused}) {
5044         push @$cmd, '-S';
5045     }
5046
5047     # host pci devices
5048     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5049       my $d = parse_hostpci($conf->{"hostpci$i"});
5050       next if !$d;
5051       my $pcidevices = $d->{pciid};
5052       foreach my $pcidevice (@$pcidevices) {
5053             my $pciid = $pcidevice->{id};
5054
5055             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5056             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5057             die "no pci device info for device '$pciid'\n" if !$info;
5058
5059             if ($d->{mdev}) {
5060                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5061                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5062             } else {
5063                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5064                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5065                 die "can't reset PCI device '$pciid'\n"
5066                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5067             }
5068       }
5069     }
5070
5071     PVE::Storage::activate_volumes($storecfg, $vollist);
5072
5073     eval {
5074         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5075             outfunc => sub {}, errfunc => sub {});
5076     };
5077     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5078     # timeout should be more than enough here...
5079     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5080
5081     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5082                                               : $defaults->{cpuunits};
5083
5084     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5085     my %run_params = (
5086         timeout => $statefile ? undef : $start_timeout,
5087         umask => 0077,
5088         noerr => 1,
5089     );
5090
5091     # when migrating, prefix QEMU output so other side can pick up any
5092     # errors that might occur and show the user
5093     if ($migratedfrom) {
5094         $run_params{quiet} = 1;
5095         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5096     }
5097
5098     my %properties = (
5099         Slice => 'qemu.slice',
5100         KillMode => 'none'
5101     );
5102
5103     if (PVE::CGroup::cgroup_mode() == 2) {
5104         $properties{CPUWeight} = $cpuunits;
5105     } else {
5106         $properties{CPUShares} = $cpuunits;
5107     }
5108
5109     if (my $cpulimit = $conf->{cpulimit}) {
5110         $properties{CPUQuota} = int($cpulimit * 100);
5111     }
5112     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5113
5114     my $run_qemu = sub {
5115         PVE::Tools::run_fork sub {
5116             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5117
5118             my $exitcode = run_command($cmd, %run_params);
5119             die "QEMU exited with code $exitcode\n" if $exitcode;
5120         };
5121     };
5122
5123     if ($conf->{hugepages}) {
5124
5125         my $code = sub {
5126             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5127             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5128
5129             PVE::QemuServer::Memory::hugepages_mount();
5130             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5131
5132             eval { $run_qemu->() };
5133             if (my $err = $@) {
5134                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5135                     if !$conf->{keephugepages};
5136                 die $err;
5137             }
5138
5139             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5140                 if !$conf->{keephugepages};
5141         };
5142         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5143
5144     } else {
5145         eval { $run_qemu->() };
5146     }
5147
5148     if (my $err = $@) {
5149         # deactivate volumes if start fails
5150         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5151         die "start failed: $err";
5152     }
5153
5154     print "migration listens on $migrate_uri\n" if $migrate_uri;
5155     $res->{migrate_uri} = $migrate_uri;
5156
5157     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5158         eval { mon_cmd($vmid, "cont"); };
5159         warn $@ if $@;
5160     }
5161
5162     #start nbd server for storage migration
5163     if (my $nbd = $migrate_opts->{nbd}) {
5164         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5165
5166         my $migrate_storage_uri;
5167         # nbd_protocol_version > 0 for unix socket support
5168         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5169             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5170             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5171             $migrate_storage_uri = "nbd:unix:$socket_path";
5172         } else {
5173             my $nodename = nodename();
5174             my $localip = $get_migration_ip->($nodename);
5175             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5176             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5177
5178             mon_cmd($vmid, "nbd-server-start", addr => {
5179                 type => 'inet',
5180                 data => {
5181                     host => "${localip}",
5182                     port => "${storage_migrate_port}",
5183                 },
5184             });
5185             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5186             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5187         }
5188
5189         $res->{migrate_storage_uri} = $migrate_storage_uri;
5190
5191         foreach my $opt (sort keys %$nbd) {
5192             my $drivestr = $nbd->{$opt}->{drivestr};
5193             my $volid = $nbd->{$opt}->{volid};
5194             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5195             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5196             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5197             print "re-using replicated volume: $opt - $volid\n"
5198                 if $nbd->{$opt}->{replicated};
5199
5200             $res->{drives}->{$opt} = $nbd->{$opt};
5201             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5202         }
5203     }
5204
5205     if ($migratedfrom) {
5206         eval {
5207             set_migration_caps($vmid);
5208         };
5209         warn $@ if $@;
5210
5211         if ($spice_port) {
5212             print "spice listens on port $spice_port\n";
5213             $res->{spice_port} = $spice_port;
5214             if ($migrate_opts->{spice_ticket}) {
5215                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5216                     $migrate_opts->{spice_ticket});
5217                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5218             }
5219         }
5220
5221     } else {
5222         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5223             if !$statefile && $conf->{balloon};
5224
5225         foreach my $opt (keys %$conf) {
5226             next if $opt !~  m/^net\d+$/;
5227             my $nicconf = parse_net($conf->{$opt});
5228             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5229         }
5230     }
5231
5232     mon_cmd($vmid, 'qom-set',
5233                 path => "machine/peripheral/balloon0",
5234                 property => "guest-stats-polling-interval",
5235                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5236
5237     if ($resume) {
5238         print "Resumed VM, removing state\n";
5239         if (my $vmstate = $conf->{vmstate}) {
5240             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5241             PVE::Storage::vdisk_free($storecfg, $vmstate);
5242         }
5243         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5244         PVE::QemuConfig->write_config($vmid, $conf);
5245     }
5246
5247     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5248
5249     return $res;
5250 }
5251
5252 sub vm_commandline {
5253     my ($storecfg, $vmid, $snapname) = @_;
5254
5255     my $conf = PVE::QemuConfig->load_config($vmid);
5256     my $forcemachine;
5257     my $forcecpu;
5258
5259     if ($snapname) {
5260         my $snapshot = $conf->{snapshots}->{$snapname};
5261         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5262
5263         # check for machine or CPU overrides in snapshot
5264         $forcemachine = $snapshot->{runningmachine};
5265         $forcecpu = $snapshot->{runningcpu};
5266
5267         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5268
5269         $conf = $snapshot;
5270     }
5271
5272     my $defaults = load_defaults();
5273
5274     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5275         $forcemachine, $forcecpu);
5276
5277     return PVE::Tools::cmd2string($cmd);
5278 }
5279
5280 sub vm_reset {
5281     my ($vmid, $skiplock) = @_;
5282
5283     PVE::QemuConfig->lock_config($vmid, sub {
5284
5285         my $conf = PVE::QemuConfig->load_config($vmid);
5286
5287         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5288
5289         mon_cmd($vmid, "system_reset");
5290     });
5291 }
5292
5293 sub get_vm_volumes {
5294     my ($conf) = @_;
5295
5296     my $vollist = [];
5297     foreach_volid($conf, sub {
5298         my ($volid, $attr) = @_;
5299
5300         return if $volid =~ m|^/|;
5301
5302         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5303         return if !$sid;
5304
5305         push @$vollist, $volid;
5306     });
5307
5308     return $vollist;
5309 }
5310
5311 sub vm_stop_cleanup {
5312     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5313
5314     eval {
5315
5316         if (!$keepActive) {
5317             my $vollist = get_vm_volumes($conf);
5318             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5319         }
5320
5321         foreach my $ext (qw(mon qmp pid vnc qga)) {
5322             unlink "/var/run/qemu-server/${vmid}.$ext";
5323         }
5324
5325         if ($conf->{ivshmem}) {
5326             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5327             # just delete it for now, VMs which have this already open do not
5328             # are affected, but new VMs will get a separated one. If this
5329             # becomes an issue we either add some sort of ref-counting or just
5330             # add a "don't delete on stop" flag to the ivshmem format.
5331             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5332         }
5333
5334         foreach my $key (keys %$conf) {
5335             next if $key !~ m/^hostpci(\d+)$/;
5336             my $hostpciindex = $1;
5337             my $d = parse_hostpci($conf->{$key});
5338             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5339
5340             foreach my $pci (@{$d->{pciid}}) {
5341                 my $pciid = $pci->{id};
5342                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5343             }
5344         }
5345
5346         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5347     };
5348     warn $@ if $@; # avoid errors - just warn
5349 }
5350
5351 # call only in locked context
5352 sub _do_vm_stop {
5353     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5354
5355     my $pid = check_running($vmid, $nocheck);
5356     return if !$pid;
5357
5358     my $conf;
5359     if (!$nocheck) {
5360         $conf = PVE::QemuConfig->load_config($vmid);
5361         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5362         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5363             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5364             $timeout = $opts->{down} if $opts->{down};
5365         }
5366         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5367     }
5368
5369     eval {
5370         if ($shutdown) {
5371             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5372                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5373             } else {
5374                 mon_cmd($vmid, "system_powerdown");
5375             }
5376         } else {
5377             mon_cmd($vmid, "quit");
5378         }
5379     };
5380     my $err = $@;
5381
5382     if (!$err) {
5383         $timeout = 60 if !defined($timeout);
5384
5385         my $count = 0;
5386         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5387             $count++;
5388             sleep 1;
5389         }
5390
5391         if ($count >= $timeout) {
5392             if ($force) {
5393                 warn "VM still running - terminating now with SIGTERM\n";
5394                 kill 15, $pid;
5395             } else {
5396                 die "VM quit/powerdown failed - got timeout\n";
5397             }
5398         } else {
5399             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5400             return;
5401         }
5402     } else {
5403        if (!check_running($vmid, $nocheck)) {
5404             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5405             return;
5406        }
5407        if ($force) {
5408             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5409             kill 15, $pid;
5410         } else {
5411             die "VM quit/powerdown failed\n";
5412         }
5413     }
5414
5415     # wait again
5416     $timeout = 10;
5417
5418     my $count = 0;
5419     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5420         $count++;
5421         sleep 1;
5422     }
5423
5424     if ($count >= $timeout) {
5425         warn "VM still running - terminating now with SIGKILL\n";
5426         kill 9, $pid;
5427         sleep 1;
5428     }
5429
5430     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5431 }
5432
5433 # Note: use $nocheck to skip tests if VM configuration file exists.
5434 # We need that when migration VMs to other nodes (files already moved)
5435 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5436 sub vm_stop {
5437     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5438
5439     $force = 1 if !defined($force) && !$shutdown;
5440
5441     if ($migratedfrom){
5442         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5443         kill 15, $pid if $pid;
5444         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5445         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5446         return;
5447     }
5448
5449     PVE::QemuConfig->lock_config($vmid, sub {
5450         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5451    });
5452 }
5453
5454 sub vm_reboot {
5455     my ($vmid, $timeout) = @_;
5456
5457     PVE::QemuConfig->lock_config($vmid, sub {
5458         eval {
5459
5460             # only reboot if running, as qmeventd starts it again on a stop event
5461             return if !check_running($vmid);
5462
5463             create_reboot_request($vmid);
5464
5465             my $storecfg = PVE::Storage::config();
5466             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5467
5468         };
5469         if (my $err = $@) {
5470             # avoid that the next normal shutdown will be confused for a reboot
5471             clear_reboot_request($vmid);
5472             die $err;
5473         }
5474    });
5475 }
5476
5477 # note: if using the statestorage parameter, the caller has to check privileges
5478 sub vm_suspend {
5479     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5480
5481     my $conf;
5482     my $path;
5483     my $storecfg;
5484     my $vmstate;
5485
5486     PVE::QemuConfig->lock_config($vmid, sub {
5487
5488         $conf = PVE::QemuConfig->load_config($vmid);
5489
5490         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5491         PVE::QemuConfig->check_lock($conf)
5492             if !($skiplock || $is_backing_up);
5493
5494         die "cannot suspend to disk during backup\n"
5495             if $is_backing_up && $includestate;
5496
5497         if ($includestate) {
5498             $conf->{lock} = 'suspending';
5499             my $date = strftime("%Y-%m-%d", localtime(time()));
5500             $storecfg = PVE::Storage::config();
5501             if (!$statestorage) {
5502                 $statestorage = find_vmstate_storage($conf, $storecfg);
5503                 # check permissions for the storage
5504                 my $rpcenv = PVE::RPCEnvironment::get();
5505                 if ($rpcenv->{type} ne 'cli') {
5506                     my $authuser = $rpcenv->get_user();
5507                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5508                 }
5509             }
5510
5511
5512             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5513                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5514             $path = PVE::Storage::path($storecfg, $vmstate);
5515             PVE::QemuConfig->write_config($vmid, $conf);
5516         } else {
5517             mon_cmd($vmid, "stop");
5518         }
5519     });
5520
5521     if ($includestate) {
5522         # save vm state
5523         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5524
5525         eval {
5526             mon_cmd($vmid, "savevm-start", statefile => $path);
5527             for(;;) {
5528                 my $state = mon_cmd($vmid, "query-savevm");
5529                 if (!$state->{status}) {
5530                     die "savevm not active\n";
5531                 } elsif ($state->{status} eq 'active') {
5532                     sleep(1);
5533                     next;
5534                 } elsif ($state->{status} eq 'completed') {
5535                     print "State saved, quitting\n";
5536                     last;
5537                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5538                     die "query-savevm failed with error '$state->{error}'\n"
5539                 } else {
5540                     die "query-savevm returned status '$state->{status}'\n";
5541                 }
5542             }
5543         };
5544         my $err = $@;
5545
5546         PVE::QemuConfig->lock_config($vmid, sub {
5547             $conf = PVE::QemuConfig->load_config($vmid);
5548             if ($err) {
5549                 # cleanup, but leave suspending lock, to indicate something went wrong
5550                 eval {
5551                     mon_cmd($vmid, "savevm-end");
5552                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5553                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5554                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5555                     PVE::QemuConfig->write_config($vmid, $conf);
5556                 };
5557                 warn $@ if $@;
5558                 die $err;
5559             }
5560
5561             die "lock changed unexpectedly\n"
5562                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5563
5564             mon_cmd($vmid, "quit");
5565             $conf->{lock} = 'suspended';
5566             PVE::QemuConfig->write_config($vmid, $conf);
5567         });
5568     }
5569 }
5570
5571 sub vm_resume {
5572     my ($vmid, $skiplock, $nocheck) = @_;
5573
5574     PVE::QemuConfig->lock_config($vmid, sub {
5575         my $res = mon_cmd($vmid, 'query-status');
5576         my $resume_cmd = 'cont';
5577         my $reset = 0;
5578
5579         if ($res->{status}) {
5580             return if $res->{status} eq 'running'; # job done, go home
5581             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5582             $reset = 1 if $res->{status} eq 'shutdown';
5583         }
5584
5585         if (!$nocheck) {
5586
5587             my $conf = PVE::QemuConfig->load_config($vmid);
5588
5589             PVE::QemuConfig->check_lock($conf)
5590                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5591         }
5592
5593         if ($reset) {
5594             # required if a VM shuts down during a backup and we get a resume
5595             # request before the backup finishes for example
5596             mon_cmd($vmid, "system_reset");
5597         }
5598         mon_cmd($vmid, $resume_cmd);
5599     });
5600 }
5601
5602 sub vm_sendkey {
5603     my ($vmid, $skiplock, $key) = @_;
5604
5605     PVE::QemuConfig->lock_config($vmid, sub {
5606
5607         my $conf = PVE::QemuConfig->load_config($vmid);
5608
5609         # there is no qmp command, so we use the human monitor command
5610         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5611         die $res if $res ne '';
5612     });
5613 }
5614
5615 # vzdump restore implementaion
5616
5617 sub tar_archive_read_firstfile {
5618     my $archive = shift;
5619
5620     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5621
5622     # try to detect archive type first
5623     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5624         die "unable to open file '$archive'\n";
5625     my $firstfile = <$fh>;
5626     kill 15, $pid;
5627     close $fh;
5628
5629     die "ERROR: archive contaions no data\n" if !$firstfile;
5630     chomp $firstfile;
5631
5632     return $firstfile;
5633 }
5634
5635 sub tar_restore_cleanup {
5636     my ($storecfg, $statfile) = @_;
5637
5638     print STDERR "starting cleanup\n";
5639
5640     if (my $fd = IO::File->new($statfile, "r")) {
5641         while (defined(my $line = <$fd>)) {
5642             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5643                 my $volid = $2;
5644                 eval {
5645                     if ($volid =~ m|^/|) {
5646                         unlink $volid || die 'unlink failed\n';
5647                     } else {
5648                         PVE::Storage::vdisk_free($storecfg, $volid);
5649                     }
5650                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5651                 };
5652                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5653             } else {
5654                 print STDERR "unable to parse line in statfile - $line";
5655             }
5656         }
5657         $fd->close();
5658     }
5659 }
5660
5661 sub restore_file_archive {
5662     my ($archive, $vmid, $user, $opts) = @_;
5663
5664     return restore_vma_archive($archive, $vmid, $user, $opts)
5665         if $archive eq '-';
5666
5667     my $info = PVE::Storage::archive_info($archive);
5668     my $format = $opts->{format} // $info->{format};
5669     my $comp = $info->{compression};
5670
5671     # try to detect archive format
5672     if ($format eq 'tar') {
5673         return restore_tar_archive($archive, $vmid, $user, $opts);
5674     } else {
5675         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5676     }
5677 }
5678
5679 # hepler to remove disks that will not be used after restore
5680 my $restore_cleanup_oldconf = sub {
5681     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5682
5683     PVE::QemuConfig->foreach_volume($oldconf, sub {
5684         my ($ds, $drive) = @_;
5685
5686         return if drive_is_cdrom($drive, 1);
5687
5688         my $volid = $drive->{file};
5689         return if !$volid || $volid =~ m|^/|;
5690
5691         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5692         return if !$path || !$owner || ($owner != $vmid);
5693
5694         # Note: only delete disk we want to restore
5695         # other volumes will become unused
5696         if ($virtdev_hash->{$ds}) {
5697             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5698             if (my $err = $@) {
5699                 warn $err;
5700             }
5701         }
5702     });
5703
5704     # delete vmstate files, after the restore we have no snapshots anymore
5705     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5706         my $snap = $oldconf->{snapshots}->{$snapname};
5707         if ($snap->{vmstate}) {
5708             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5709             if (my $err = $@) {
5710                 warn $err;
5711             }
5712         }
5713     }
5714 };
5715
5716 # Helper to parse vzdump backup device hints
5717 #
5718 # $rpcenv: Environment, used to ckeck storage permissions
5719 # $user: User ID, to check storage permissions
5720 # $storecfg: Storage configuration
5721 # $fh: the file handle for reading the configuration
5722 # $devinfo: should contain device sizes for all backu-up'ed devices
5723 # $options: backup options (pool, default storage)
5724 #
5725 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5726 my $parse_backup_hints = sub {
5727     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5728
5729     my $virtdev_hash = {};
5730
5731     while (defined(my $line = <$fh>)) {
5732         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5733             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5734             die "archive does not contain data for drive '$virtdev'\n"
5735                 if !$devinfo->{$devname};
5736
5737             if (defined($options->{storage})) {
5738                 $storeid = $options->{storage} || 'local';
5739             } elsif (!$storeid) {
5740                 $storeid = 'local';
5741             }
5742             $format = 'raw' if !$format;
5743             $devinfo->{$devname}->{devname} = $devname;
5744             $devinfo->{$devname}->{virtdev} = $virtdev;
5745             $devinfo->{$devname}->{format} = $format;
5746             $devinfo->{$devname}->{storeid} = $storeid;
5747
5748             # check permission on storage
5749             my $pool = $options->{pool}; # todo: do we need that?
5750             if ($user ne 'root@pam') {
5751                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5752             }
5753
5754             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5755         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5756             my $virtdev = $1;
5757             my $drive = parse_drive($virtdev, $2);
5758             if (drive_is_cloudinit($drive)) {
5759                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5760                 $storeid = $options->{storage} if defined ($options->{storage});
5761                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5762                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5763
5764                 $virtdev_hash->{$virtdev} = {
5765                     format => $format,
5766                     storeid => $storeid,
5767                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5768                     is_cloudinit => 1,
5769                 };
5770             }
5771         }
5772     }
5773
5774     return $virtdev_hash;
5775 };
5776
5777 # Helper to allocate and activate all volumes required for a restore
5778 #
5779 # $storecfg: Storage configuration
5780 # $virtdev_hash: as returned by parse_backup_hints()
5781 #
5782 # Returns: { $virtdev => $volid }
5783 my $restore_allocate_devices = sub {
5784     my ($storecfg, $virtdev_hash, $vmid) = @_;
5785
5786     my $map = {};
5787     foreach my $virtdev (sort keys %$virtdev_hash) {
5788         my $d = $virtdev_hash->{$virtdev};
5789         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5790         my $storeid = $d->{storeid};
5791         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5792
5793         # test if requested format is supported
5794         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5795         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5796         $d->{format} = $defFormat if !$supported;
5797
5798         my $name;
5799         if ($d->{is_cloudinit}) {
5800             $name = "vm-$vmid-cloudinit";
5801             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5802             if ($scfg->{path}) {
5803                 $name .= ".$d->{format}";
5804             }
5805         }
5806
5807         my $volid = PVE::Storage::vdisk_alloc(
5808             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5809
5810         print STDERR "new volume ID is '$volid'\n";
5811         $d->{volid} = $volid;
5812
5813         PVE::Storage::activate_volumes($storecfg, [$volid]);
5814
5815         $map->{$virtdev} = $volid;
5816     }
5817
5818     return $map;
5819 };
5820
5821 my $restore_update_config_line = sub {
5822     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5823
5824     return if $line =~ m/^\#qmdump\#/;
5825     return if $line =~ m/^\#vzdump\#/;
5826     return if $line =~ m/^lock:/;
5827     return if $line =~ m/^unused\d+:/;
5828     return if $line =~ m/^parent:/;
5829
5830     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5831     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5832         # try to convert old 1.X settings
5833         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5834         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5835             my ($model, $macaddr) = split(/\=/, $devconfig);
5836             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5837             my $net = {
5838                 model => $model,
5839                 bridge => "vmbr$ind",
5840                 macaddr => $macaddr,
5841             };
5842             my $netstr = print_net($net);
5843
5844             print $outfd "net$cookie->{netcount}: $netstr\n";
5845             $cookie->{netcount}++;
5846         }
5847     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5848         my ($id, $netstr) = ($1, $2);
5849         my $net = parse_net($netstr);
5850         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5851         $netstr = print_net($net);
5852         print $outfd "$id: $netstr\n";
5853     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5854         my $virtdev = $1;
5855         my $value = $3;
5856         my $di = parse_drive($virtdev, $value);
5857         if (defined($di->{backup}) && !$di->{backup}) {
5858             print $outfd "#$line";
5859         } elsif ($map->{$virtdev}) {
5860             delete $di->{format}; # format can change on restore
5861             $di->{file} = $map->{$virtdev};
5862             $value = print_drive($di);
5863             print $outfd "$virtdev: $value\n";
5864         } else {
5865             print $outfd $line;
5866         }
5867     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5868         my $vmgenid = $1;
5869         if ($vmgenid ne '0') {
5870             # always generate a new vmgenid if there was a valid one setup
5871             $vmgenid = generate_uuid();
5872         }
5873         print $outfd "vmgenid: $vmgenid\n";
5874     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5875         my ($uuid, $uuid_str);
5876         UUID::generate($uuid);
5877         UUID::unparse($uuid, $uuid_str);
5878         my $smbios1 = parse_smbios1($2);
5879         $smbios1->{uuid} = $uuid_str;
5880         print $outfd $1.print_smbios1($smbios1)."\n";
5881     } else {
5882         print $outfd $line;
5883     }
5884 };
5885
5886 my $restore_deactivate_volumes = sub {
5887     my ($storecfg, $devinfo) = @_;
5888
5889     my $vollist = [];
5890     foreach my $devname (keys %$devinfo) {
5891         my $volid = $devinfo->{$devname}->{volid};
5892         push @$vollist, $volid if $volid;
5893     }
5894
5895     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5896 };
5897
5898 my $restore_destroy_volumes = sub {
5899     my ($storecfg, $devinfo) = @_;
5900
5901     foreach my $devname (keys %$devinfo) {
5902         my $volid = $devinfo->{$devname}->{volid};
5903         next if !$volid;
5904         eval {
5905             if ($volid =~ m|^/|) {
5906                 unlink $volid || die 'unlink failed\n';
5907             } else {
5908                 PVE::Storage::vdisk_free($storecfg, $volid);
5909             }
5910             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5911         };
5912         print STDERR "unable to cleanup '$volid' - $@" if $@;
5913     }
5914 };
5915
5916 sub scan_volids {
5917     my ($cfg, $vmid) = @_;
5918
5919     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5920
5921     my $volid_hash = {};
5922     foreach my $storeid (keys %$info) {
5923         foreach my $item (@{$info->{$storeid}}) {
5924             next if !($item->{volid} && $item->{size});
5925             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5926             $volid_hash->{$item->{volid}} = $item;
5927         }
5928     }
5929
5930     return $volid_hash;
5931 }
5932
5933 sub update_disk_config {
5934     my ($vmid, $conf, $volid_hash) = @_;
5935
5936     my $changes;
5937     my $prefix = "VM $vmid";
5938
5939     # used and unused disks
5940     my $referenced = {};
5941
5942     # Note: it is allowed to define multiple storages with same path (alias), so
5943     # we need to check both 'volid' and real 'path' (two different volid can point
5944     # to the same path).
5945
5946     my $referencedpath = {};
5947
5948     # update size info
5949     PVE::QemuConfig->foreach_volume($conf, sub {
5950         my ($opt, $drive) = @_;
5951
5952         my $volid = $drive->{file};
5953         return if !$volid;
5954         my $volume = $volid_hash->{$volid};
5955
5956         # mark volid as "in-use" for next step
5957         $referenced->{$volid} = 1;
5958         if ($volume && (my $path = $volume->{path})) {
5959             $referencedpath->{$path} = 1;
5960         }
5961
5962         return if drive_is_cdrom($drive);
5963         return if !$volume;
5964
5965         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5966         if (defined($updated)) {
5967             $changes = 1;
5968             $conf->{$opt} = print_drive($updated);
5969             print "$prefix ($opt): $msg\n";
5970         }
5971     });
5972
5973     # remove 'unusedX' entry if volume is used
5974     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5975         my ($opt, $drive) = @_;
5976
5977         my $volid = $drive->{file};
5978         return if !$volid;
5979
5980         my $path;
5981         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5982         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5983             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5984             $changes = 1;
5985             delete $conf->{$opt};
5986         }
5987
5988         $referenced->{$volid} = 1;
5989         $referencedpath->{$path} = 1 if $path;
5990     });
5991
5992     foreach my $volid (sort keys %$volid_hash) {
5993         next if $volid =~ m/vm-$vmid-state-/;
5994         next if $referenced->{$volid};
5995         my $path = $volid_hash->{$volid}->{path};
5996         next if !$path; # just to be sure
5997         next if $referencedpath->{$path};
5998         $changes = 1;
5999         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6000         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6001         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6002     }
6003
6004     return $changes;
6005 }
6006
6007 sub rescan {
6008     my ($vmid, $nolock, $dryrun) = @_;
6009
6010     my $cfg = PVE::Storage::config();
6011
6012     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6013     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6014     foreach my $stor (keys %{$cfg->{ids}}) {
6015         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6016     }
6017
6018     print "rescan volumes...\n";
6019     my $volid_hash = scan_volids($cfg, $vmid);
6020
6021     my $updatefn =  sub {
6022         my ($vmid) = @_;
6023
6024         my $conf = PVE::QemuConfig->load_config($vmid);
6025
6026         PVE::QemuConfig->check_lock($conf);
6027
6028         my $vm_volids = {};
6029         foreach my $volid (keys %$volid_hash) {
6030             my $info = $volid_hash->{$volid};
6031             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6032         }
6033
6034         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6035
6036         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6037     };
6038
6039     if (defined($vmid)) {
6040         if ($nolock) {
6041             &$updatefn($vmid);
6042         } else {
6043             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6044         }
6045     } else {
6046         my $vmlist = config_list();
6047         foreach my $vmid (keys %$vmlist) {
6048             if ($nolock) {
6049                 &$updatefn($vmid);
6050             } else {
6051                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6052             }
6053         }
6054     }
6055 }
6056
6057 sub restore_proxmox_backup_archive {
6058     my ($archive, $vmid, $user, $options) = @_;
6059
6060     my $storecfg = PVE::Storage::config();
6061
6062     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6063     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6064
6065     my $fingerprint = $scfg->{fingerprint};
6066     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6067
6068     my $repo = PVE::PBSClient::get_repository($scfg);
6069
6070     # This is only used for `pbs-restore`!
6071     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6072     local $ENV{PBS_PASSWORD} = $password;
6073     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6074
6075     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6076         PVE::Storage::parse_volname($storecfg, $archive);
6077
6078     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6079
6080     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6081
6082     my $tmpdir = "/var/tmp/vzdumptmp$$";
6083     rmtree $tmpdir;
6084     mkpath $tmpdir;
6085
6086     my $conffile = PVE::QemuConfig->config_file($vmid);
6087     my $tmpfn = "$conffile.$$.tmp";
6088      # disable interrupts (always do cleanups)
6089     local $SIG{INT} =
6090         local $SIG{TERM} =
6091         local $SIG{QUIT} =
6092         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6093
6094     # Note: $oldconf is undef if VM does not exists
6095     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6096     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6097
6098     my $rpcenv = PVE::RPCEnvironment::get();
6099     my $devinfo = {};
6100
6101     eval {
6102         # enable interrupts
6103         local $SIG{INT} =
6104             local $SIG{TERM} =
6105             local $SIG{QUIT} =
6106             local $SIG{HUP} =
6107             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6108
6109         my $cfgfn = "$tmpdir/qemu-server.conf";
6110         my $firewall_config_fn = "$tmpdir/fw.conf";
6111         my $index_fn = "$tmpdir/index.json";
6112
6113         my $cmd = "restore";
6114
6115         my $param = [$pbs_backup_name, "index.json", $index_fn];
6116         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6117         my $index = PVE::Tools::file_get_contents($index_fn);
6118         $index = decode_json($index);
6119
6120         # print Dumper($index);
6121         foreach my $info (@{$index->{files}}) {
6122             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6123                 my $devname = $1;
6124                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6125                     $devinfo->{$devname}->{size} = $1;
6126                 } else {
6127                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6128                 }
6129             }
6130         }
6131
6132         my $is_qemu_server_backup = scalar(
6133             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6134         );
6135         if (!$is_qemu_server_backup) {
6136             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6137         }
6138         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6139
6140         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6141         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6142
6143         if ($has_firewall_config) {
6144             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6145             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6146
6147             my $pve_firewall_dir = '/etc/pve/firewall';
6148             mkdir $pve_firewall_dir; # make sure the dir exists
6149             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6150         }
6151
6152         my $fh = IO::File->new($cfgfn, "r") ||
6153             die "unable to read qemu-server.conf - $!\n";
6154
6155         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6156
6157         # fixme: rate limit?
6158
6159         # create empty/temp config
6160         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6161
6162         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6163
6164         # allocate volumes
6165         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6166
6167         foreach my $virtdev (sort keys %$virtdev_hash) {
6168             my $d = $virtdev_hash->{$virtdev};
6169             next if $d->{is_cloudinit}; # no need to restore cloudinit
6170
6171             my $volid = $d->{volid};
6172
6173             my $path = PVE::Storage::path($storecfg, $volid);
6174
6175             # This is the ONLY user of the PBS_ env vars set on top of this function!
6176             my $pbs_restore_cmd = [
6177                 '/usr/bin/pbs-restore',
6178                 '--repository', $repo,
6179                 $pbs_backup_name,
6180                 "$d->{devname}.img.fidx",
6181                 $path,
6182                 '--verbose',
6183                 ];
6184
6185             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6186             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6187
6188             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6189                 push @$pbs_restore_cmd, '--skip-zero';
6190             }
6191
6192             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6193             print "restore proxmox backup image: $dbg_cmdstring\n";
6194             run_command($pbs_restore_cmd);
6195         }
6196
6197         $fh->seek(0, 0) || die "seek failed - $!\n";
6198
6199         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6200
6201         my $cookie = { netcount => 0 };
6202         while (defined(my $line = <$fh>)) {
6203             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6204         }
6205
6206         $fh->close();
6207         $outfd->close();
6208     };
6209     my $err = $@;
6210
6211     $restore_deactivate_volumes->($storecfg, $devinfo);
6212
6213     rmtree $tmpdir;
6214
6215     if ($err) {
6216         unlink $tmpfn;
6217         $restore_destroy_volumes->($storecfg, $devinfo);
6218         die $err;
6219     }
6220
6221     rename($tmpfn, $conffile) ||
6222         die "unable to commit configuration file '$conffile'\n";
6223
6224     PVE::Cluster::cfs_update(); # make sure we read new file
6225
6226     eval { rescan($vmid, 1); };
6227     warn $@ if $@;
6228 }
6229
6230 sub restore_vma_archive {
6231     my ($archive, $vmid, $user, $opts, $comp) = @_;
6232
6233     my $readfrom = $archive;
6234
6235     my $cfg = PVE::Storage::config();
6236     my $commands = [];
6237     my $bwlimit = $opts->{bwlimit};
6238
6239     my $dbg_cmdstring = '';
6240     my $add_pipe = sub {
6241         my ($cmd) = @_;
6242         push @$commands, $cmd;
6243         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6244         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6245         $readfrom = '-';
6246     };
6247
6248     my $input = undef;
6249     if ($archive eq '-') {
6250         $input = '<&STDIN';
6251     } else {
6252         # If we use a backup from a PVE defined storage we also consider that
6253         # storage's rate limit:
6254         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6255         if (defined($volid)) {
6256             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6257             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6258             if ($readlimit) {
6259                 print STDERR "applying read rate limit: $readlimit\n";
6260                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6261                 $add_pipe->($cstream);
6262             }
6263         }
6264     }
6265
6266     if ($comp) {
6267         my $info = PVE::Storage::decompressor_info('vma', $comp);
6268         my $cmd = $info->{decompressor};
6269         push @$cmd, $readfrom;
6270         $add_pipe->($cmd);
6271     }
6272
6273     my $tmpdir = "/var/tmp/vzdumptmp$$";
6274     rmtree $tmpdir;
6275
6276     # disable interrupts (always do cleanups)
6277     local $SIG{INT} =
6278         local $SIG{TERM} =
6279         local $SIG{QUIT} =
6280         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6281
6282     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6283     POSIX::mkfifo($mapfifo, 0600);
6284     my $fifofh;
6285     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6286
6287     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6288
6289     my $oldtimeout;
6290     my $timeout = 5;
6291
6292     my $devinfo = {};
6293
6294     my $rpcenv = PVE::RPCEnvironment::get();
6295
6296     my $conffile = PVE::QemuConfig->config_file($vmid);
6297     my $tmpfn = "$conffile.$$.tmp";
6298
6299     # Note: $oldconf is undef if VM does not exist
6300     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6301     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6302
6303     my %storage_limits;
6304
6305     my $print_devmap = sub {
6306         my $cfgfn = "$tmpdir/qemu-server.conf";
6307
6308         # we can read the config - that is already extracted
6309         my $fh = IO::File->new($cfgfn, "r") ||
6310             die "unable to read qemu-server.conf - $!\n";
6311
6312         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6313         if (-f $fwcfgfn) {
6314             my $pve_firewall_dir = '/etc/pve/firewall';
6315             mkdir $pve_firewall_dir; # make sure the dir exists
6316             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6317         }
6318
6319         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6320
6321         foreach my $key (keys %storage_limits) {
6322             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6323             next if !$limit;
6324             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6325             $storage_limits{$key} = $limit * 1024;
6326         }
6327
6328         foreach my $devname (keys %$devinfo) {
6329             die "found no device mapping information for device '$devname'\n"
6330                 if !$devinfo->{$devname}->{virtdev};
6331         }
6332
6333         # create empty/temp config
6334         if ($oldconf) {
6335             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6336             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6337         }
6338
6339         # allocate volumes
6340         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6341
6342         # print restore information to $fifofh
6343         foreach my $virtdev (sort keys %$virtdev_hash) {
6344             my $d = $virtdev_hash->{$virtdev};
6345             next if $d->{is_cloudinit}; # no need to restore cloudinit
6346
6347             my $storeid = $d->{storeid};
6348             my $volid = $d->{volid};
6349
6350             my $map_opts = '';
6351             if (my $limit = $storage_limits{$storeid}) {
6352                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6353             }
6354
6355             my $write_zeros = 1;
6356             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6357                 $write_zeros = 0;
6358             }
6359
6360             my $path = PVE::Storage::path($cfg, $volid);
6361
6362             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6363
6364             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6365         }
6366
6367         $fh->seek(0, 0) || die "seek failed - $!\n";
6368
6369         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6370
6371         my $cookie = { netcount => 0 };
6372         while (defined(my $line = <$fh>)) {
6373             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6374         }
6375
6376         $fh->close();
6377         $outfd->close();
6378     };
6379
6380     eval {
6381         # enable interrupts
6382         local $SIG{INT} =
6383             local $SIG{TERM} =
6384             local $SIG{QUIT} =
6385             local $SIG{HUP} =
6386             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6387         local $SIG{ALRM} = sub { die "got timeout\n"; };
6388
6389         $oldtimeout = alarm($timeout);
6390
6391         my $parser = sub {
6392             my $line = shift;
6393
6394             print "$line\n";
6395
6396             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6397                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6398                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6399             } elsif ($line =~ m/^CTIME: /) {
6400                 # we correctly received the vma config, so we can disable
6401                 # the timeout now for disk allocation (set to 10 minutes, so
6402                 # that we always timeout if something goes wrong)
6403                 alarm(600);
6404                 &$print_devmap();
6405                 print $fifofh "done\n";
6406                 my $tmp = $oldtimeout || 0;
6407                 $oldtimeout = undef;
6408                 alarm($tmp);
6409                 close($fifofh);
6410                 $fifofh = undef;
6411             }
6412         };
6413
6414         print "restore vma archive: $dbg_cmdstring\n";
6415         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6416     };
6417     my $err = $@;
6418
6419     alarm($oldtimeout) if $oldtimeout;
6420
6421     $restore_deactivate_volumes->($cfg, $devinfo);
6422
6423     close($fifofh) if $fifofh;
6424     unlink $mapfifo;
6425     rmtree $tmpdir;
6426
6427     if ($err) {
6428         unlink $tmpfn;
6429         $restore_destroy_volumes->($cfg, $devinfo);
6430         die $err;
6431     }
6432
6433     rename($tmpfn, $conffile) ||
6434         die "unable to commit configuration file '$conffile'\n";
6435
6436     PVE::Cluster::cfs_update(); # make sure we read new file
6437
6438     eval { rescan($vmid, 1); };
6439     warn $@ if $@;
6440 }
6441
6442 sub restore_tar_archive {
6443     my ($archive, $vmid, $user, $opts) = @_;
6444
6445     if ($archive ne '-') {
6446         my $firstfile = tar_archive_read_firstfile($archive);
6447         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6448             if $firstfile ne 'qemu-server.conf';
6449     }
6450
6451     my $storecfg = PVE::Storage::config();
6452
6453     # avoid zombie disks when restoring over an existing VM -> cleanup first
6454     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6455     # skiplock=1 because qmrestore has set the 'create' lock itself already
6456     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6457     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6458
6459     my $tocmd = "/usr/lib/qemu-server/qmextract";
6460
6461     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6462     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6463     $tocmd .= ' --prealloc' if $opts->{prealloc};
6464     $tocmd .= ' --info' if $opts->{info};
6465
6466     # tar option "xf" does not autodetect compression when read from STDIN,
6467     # so we pipe to zcat
6468     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6469         PVE::Tools::shellquote("--to-command=$tocmd");
6470
6471     my $tmpdir = "/var/tmp/vzdumptmp$$";
6472     mkpath $tmpdir;
6473
6474     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6475     local $ENV{VZDUMP_VMID} = $vmid;
6476     local $ENV{VZDUMP_USER} = $user;
6477
6478     my $conffile = PVE::QemuConfig->config_file($vmid);
6479     my $tmpfn = "$conffile.$$.tmp";
6480
6481     # disable interrupts (always do cleanups)
6482     local $SIG{INT} =
6483         local $SIG{TERM} =
6484         local $SIG{QUIT} =
6485         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6486
6487     eval {
6488         # enable interrupts
6489         local $SIG{INT} =
6490             local $SIG{TERM} =
6491             local $SIG{QUIT} =
6492             local $SIG{HUP} =
6493             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6494
6495         if ($archive eq '-') {
6496             print "extracting archive from STDIN\n";
6497             run_command($cmd, input => "<&STDIN");
6498         } else {
6499             print "extracting archive '$archive'\n";
6500             run_command($cmd);
6501         }
6502
6503         return if $opts->{info};
6504
6505         # read new mapping
6506         my $map = {};
6507         my $statfile = "$tmpdir/qmrestore.stat";
6508         if (my $fd = IO::File->new($statfile, "r")) {
6509             while (defined (my $line = <$fd>)) {
6510                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6511                     $map->{$1} = $2 if $1;
6512                 } else {
6513                     print STDERR "unable to parse line in statfile - $line\n";
6514                 }
6515             }
6516             $fd->close();
6517         }
6518
6519         my $confsrc = "$tmpdir/qemu-server.conf";
6520
6521         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6522
6523         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6524
6525         my $cookie = { netcount => 0 };
6526         while (defined (my $line = <$srcfd>)) {
6527             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6528         }
6529
6530         $srcfd->close();
6531         $outfd->close();
6532     };
6533     if (my $err = $@) {
6534         unlink $tmpfn;
6535         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6536         die $err;
6537     }
6538
6539     rmtree $tmpdir;
6540
6541     rename $tmpfn, $conffile ||
6542         die "unable to commit configuration file '$conffile'\n";
6543
6544     PVE::Cluster::cfs_update(); # make sure we read new file
6545
6546     eval { rescan($vmid, 1); };
6547     warn $@ if $@;
6548 };
6549
6550 sub foreach_storage_used_by_vm {
6551     my ($conf, $func) = @_;
6552
6553     my $sidhash = {};
6554
6555     PVE::QemuConfig->foreach_volume($conf, sub {
6556         my ($ds, $drive) = @_;
6557         return if drive_is_cdrom($drive);
6558
6559         my $volid = $drive->{file};
6560
6561         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6562         $sidhash->{$sid} = $sid if $sid;
6563     });
6564
6565     foreach my $sid (sort keys %$sidhash) {
6566         &$func($sid);
6567     }
6568 }
6569
6570 my $qemu_snap_storage = {
6571     rbd => 1,
6572 };
6573 sub do_snapshots_with_qemu {
6574     my ($storecfg, $volid) = @_;
6575
6576     my $storage_name = PVE::Storage::parse_volume_id($volid);
6577     my $scfg = $storecfg->{ids}->{$storage_name};
6578     die "could not find storage '$storage_name'\n" if !defined($scfg);
6579
6580     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6581         return 1;
6582     }
6583
6584     if ($volid =~ m/\.(qcow2|qed)$/){
6585         return 1;
6586     }
6587
6588     return;
6589 }
6590
6591 sub qga_check_running {
6592     my ($vmid, $nowarn) = @_;
6593
6594     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6595     if ($@) {
6596         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6597         return 0;
6598     }
6599     return 1;
6600 }
6601
6602 sub template_create {
6603     my ($vmid, $conf, $disk) = @_;
6604
6605     my $storecfg = PVE::Storage::config();
6606
6607     PVE::QemuConfig->foreach_volume($conf, sub {
6608         my ($ds, $drive) = @_;
6609
6610         return if drive_is_cdrom($drive);
6611         return if $disk && $ds ne $disk;
6612
6613         my $volid = $drive->{file};
6614         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6615
6616         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6617         $drive->{file} = $voliddst;
6618         $conf->{$ds} = print_drive($drive);
6619         PVE::QemuConfig->write_config($vmid, $conf);
6620     });
6621 }
6622
6623 sub convert_iscsi_path {
6624     my ($path) = @_;
6625
6626     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6627         my $portal = $1;
6628         my $target = $2;
6629         my $lun = $3;
6630
6631         my $initiator_name = get_initiator_name();
6632
6633         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6634                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6635     }
6636
6637     die "cannot convert iscsi path '$path', unkown format\n";
6638 }
6639
6640 sub qemu_img_convert {
6641     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6642
6643     my $storecfg = PVE::Storage::config();
6644     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6645     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6646
6647     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6648
6649     my $cachemode;
6650     my $src_path;
6651     my $src_is_iscsi = 0;
6652     my $src_format;
6653
6654     if ($src_storeid) {
6655         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6656         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6657         $src_format = qemu_img_format($src_scfg, $src_volname);
6658         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6659         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6660         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6661     } elsif (-f $src_volid) {
6662         $src_path = $src_volid;
6663         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6664             $src_format = $1;
6665         }
6666     }
6667
6668     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6669
6670     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6671     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6672     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6673     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6674
6675     my $cmd = [];
6676     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6677     push @$cmd, '-l', "snapshot.name=$snapname"
6678         if $snapname && $src_format && $src_format eq "qcow2";
6679     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6680     push @$cmd, '-T', $cachemode if defined($cachemode);
6681
6682     if ($src_is_iscsi) {
6683         push @$cmd, '--image-opts';
6684         $src_path = convert_iscsi_path($src_path);
6685     } elsif ($src_format) {
6686         push @$cmd, '-f', $src_format;
6687     }
6688
6689     if ($dst_is_iscsi) {
6690         push @$cmd, '--target-image-opts';
6691         $dst_path = convert_iscsi_path($dst_path);
6692     } else {
6693         push @$cmd, '-O', $dst_format;
6694     }
6695
6696     push @$cmd, $src_path;
6697
6698     if (!$dst_is_iscsi && $is_zero_initialized) {
6699         push @$cmd, "zeroinit:$dst_path";
6700     } else {
6701         push @$cmd, $dst_path;
6702     }
6703
6704     my $parser = sub {
6705         my $line = shift;
6706         if($line =~ m/\((\S+)\/100\%\)/){
6707             my $percent = $1;
6708             my $transferred = int($size * $percent / 100);
6709             my $remaining = $size - $transferred;
6710
6711             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6712         }
6713
6714     };
6715
6716     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6717     my $err = $@;
6718     die "copy failed: $err" if $err;
6719 }
6720
6721 sub qemu_img_format {
6722     my ($scfg, $volname) = @_;
6723
6724     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6725         return $1;
6726     } else {
6727         return "raw";
6728     }
6729 }
6730
6731 sub qemu_drive_mirror {
6732     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6733
6734     $jobs = {} if !$jobs;
6735
6736     my $qemu_target;
6737     my $format;
6738     $jobs->{"drive-$drive"} = {};
6739
6740     if ($dst_volid =~ /^nbd:/) {
6741         $qemu_target = $dst_volid;
6742         $format = "nbd";
6743     } else {
6744         my $storecfg = PVE::Storage::config();
6745         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6746
6747         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6748
6749         $format = qemu_img_format($dst_scfg, $dst_volname);
6750
6751         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6752
6753         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6754     }
6755
6756     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6757     $opts->{format} = $format if $format;
6758
6759     if (defined($src_bitmap)) {
6760         $opts->{sync} = 'incremental';
6761         $opts->{bitmap} = $src_bitmap;
6762         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6763     }
6764
6765     if (defined($bwlimit)) {
6766         $opts->{speed} = $bwlimit * 1024;
6767         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6768     } else {
6769         print "drive mirror is starting for drive-$drive\n";
6770     }
6771
6772     # if a job already runs for this device we get an error, catch it for cleanup
6773     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6774     if (my $err = $@) {
6775         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6776         warn "$@\n" if $@;
6777         die "mirroring error: $err\n";
6778     }
6779
6780     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6781 }
6782
6783 # $completion can be either
6784 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6785 # 'cancel': wait until all jobs are ready, block-job-cancel them
6786 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6787 sub qemu_drive_mirror_monitor {
6788     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6789
6790     $completion //= 'complete';
6791
6792     eval {
6793         my $err_complete = 0;
6794
6795         while (1) {
6796             die "storage migration timed out\n" if $err_complete > 300;
6797
6798             my $stats = mon_cmd($vmid, "query-block-jobs");
6799
6800             my $running_mirror_jobs = {};
6801             foreach my $stat (@$stats) {
6802                 next if $stat->{type} ne 'mirror';
6803                 $running_mirror_jobs->{$stat->{device}} = $stat;
6804             }
6805
6806             my $readycounter = 0;
6807
6808             foreach my $job (keys %$jobs) {
6809
6810                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6811                     print "$job : finished\n";
6812                     delete $jobs->{$job};
6813                     next;
6814                 }
6815
6816                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6817
6818                 my $busy = $running_mirror_jobs->{$job}->{busy};
6819                 my $ready = $running_mirror_jobs->{$job}->{ready};
6820                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6821                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6822                     my $remaining = $total - $transferred;
6823                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6824
6825                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6826                 }
6827
6828                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6829             }
6830
6831             last if scalar(keys %$jobs) == 0;
6832
6833             if ($readycounter == scalar(keys %$jobs)) {
6834                 print "all mirroring jobs are ready \n";
6835                 last if $completion eq 'skip'; #do the complete later
6836
6837                 if ($vmiddst && $vmiddst != $vmid) {
6838                     my $agent_running = $qga && qga_check_running($vmid);
6839                     if ($agent_running) {
6840                         print "freeze filesystem\n";
6841                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6842                     } else {
6843                         print "suspend vm\n";
6844                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6845                     }
6846
6847                     # if we clone a disk for a new target vm, we don't switch the disk
6848                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6849
6850                     if ($agent_running) {
6851                         print "unfreeze filesystem\n";
6852                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6853                     } else {
6854                         print "resume vm\n";
6855                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6856                     }
6857
6858                     last;
6859                 } else {
6860
6861                     foreach my $job (keys %$jobs) {
6862                         # try to switch the disk if source and destination are on the same guest
6863                         print "$job: Completing block job...\n";
6864
6865                         my $op;
6866                         if ($completion eq 'complete') {
6867                             $op = 'block-job-complete';
6868                         } elsif ($completion eq 'cancel') {
6869                             $op = 'block-job-cancel';
6870                         } else {
6871                             die "invalid completion value: $completion\n";
6872                         }
6873                         eval { mon_cmd($vmid, $op, device => $job) };
6874                         if ($@ =~ m/cannot be completed/) {
6875                             print "$job: Block job cannot be completed, try again.\n";
6876                             $err_complete++;
6877                         }else {
6878                             print "$job: Completed successfully.\n";
6879                             $jobs->{$job}->{complete} = 1;
6880                         }
6881                     }
6882                 }
6883             }
6884             sleep 1;
6885         }
6886     };
6887     my $err = $@;
6888
6889     if ($err) {
6890         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6891         die "mirroring error: $err";
6892     }
6893
6894 }
6895
6896 sub qemu_blockjobs_cancel {
6897     my ($vmid, $jobs) = @_;
6898
6899     foreach my $job (keys %$jobs) {
6900         print "$job: Cancelling block job\n";
6901         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6902         $jobs->{$job}->{cancel} = 1;
6903     }
6904
6905     while (1) {
6906         my $stats = mon_cmd($vmid, "query-block-jobs");
6907
6908         my $running_jobs = {};
6909         foreach my $stat (@$stats) {
6910             $running_jobs->{$stat->{device}} = $stat;
6911         }
6912
6913         foreach my $job (keys %$jobs) {
6914
6915             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6916                 print "$job: Done.\n";
6917                 delete $jobs->{$job};
6918             }
6919         }
6920
6921         last if scalar(keys %$jobs) == 0;
6922
6923         sleep 1;
6924     }
6925 }
6926
6927 sub clone_disk {
6928     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6929         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6930
6931     my $newvolid;
6932
6933     if (!$full) {
6934         print "create linked clone of drive $drivename ($drive->{file})\n";
6935         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6936         push @$newvollist, $newvolid;
6937     } else {
6938
6939         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6940         $storeid = $storage if $storage;
6941
6942         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6943
6944         print "create full clone of drive $drivename ($drive->{file})\n";
6945         my $name = undef;
6946         my $size = undef;
6947         if (drive_is_cloudinit($drive)) {
6948             $name = "vm-$newvmid-cloudinit";
6949             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6950             if ($scfg->{path}) {
6951                 $name .= ".$dst_format";
6952             }
6953             $snapname = undef;
6954             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6955         } elsif ($drivename eq 'efidisk0') {
6956             $size = get_efivars_size($conf);
6957         } else {
6958             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
6959         }
6960         $newvolid = PVE::Storage::vdisk_alloc(
6961             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
6962         );
6963         push @$newvollist, $newvolid;
6964
6965         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6966
6967         if (drive_is_cloudinit($drive)) {
6968             goto no_data_clone;
6969         }
6970
6971         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6972         if (!$running || $snapname) {
6973             # TODO: handle bwlimits
6974             if ($drivename eq 'efidisk0') {
6975                 # the relevant data on the efidisk may be smaller than the source
6976                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6977                 # that is given by the OVMF_VARS.fd
6978                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6979                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6980                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6981                     "if=$src_path", "of=$dst_path"]);
6982             } else {
6983                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6984             }
6985         } else {
6986
6987             my $kvmver = get_running_qemu_version ($vmid);
6988             if (!min_version($kvmver, 2, 7)) {
6989                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6990                     if $drive->{iothread};
6991             }
6992
6993             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6994                 $completion, $qga, $bwlimit);
6995         }
6996     }
6997
6998 no_data_clone:
6999     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7000
7001     my $disk = $drive;
7002     $disk->{format} = undef;
7003     $disk->{file} = $newvolid;
7004     $disk->{size} = $size if defined($size);
7005
7006     return $disk;
7007 }
7008
7009 sub get_running_qemu_version {
7010     my ($vmid) = @_;
7011     my $res = mon_cmd($vmid, "query-version");
7012     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7013 }
7014
7015 sub qemu_use_old_bios_files {
7016     my ($machine_type) = @_;
7017
7018     return if !$machine_type;
7019
7020     my $use_old_bios_files = undef;
7021
7022     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7023         $machine_type = $1;
7024         $use_old_bios_files = 1;
7025     } else {
7026         my $version = extract_version($machine_type, kvm_user_version());
7027         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7028         # load new efi bios files on migration. So this hack is required to allow
7029         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7030         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7031         $use_old_bios_files = !min_version($version, 2, 4);
7032     }
7033
7034     return ($use_old_bios_files, $machine_type);
7035 }
7036
7037 sub get_efivars_size {
7038     my ($conf) = @_;
7039     my $arch = get_vm_arch($conf);
7040     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7041     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7042     return -s $ovmf_vars;
7043 }
7044
7045 sub update_efidisk_size {
7046     my ($conf) = @_;
7047
7048     return if !defined($conf->{efidisk0});
7049
7050     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7051     $disk->{size} = get_efivars_size($conf);
7052     $conf->{efidisk0} = print_drive($disk);
7053
7054     return;
7055 }
7056
7057 sub create_efidisk($$$$$) {
7058     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7059
7060     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7061     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7062
7063     my $vars_size_b = -s $ovmf_vars;
7064     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7065     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7066     PVE::Storage::activate_volumes($storecfg, [$volid]);
7067
7068     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7069     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7070
7071     return ($volid, $size/1024);
7072 }
7073
7074 sub vm_iothreads_list {
7075     my ($vmid) = @_;
7076
7077     my $res = mon_cmd($vmid, 'query-iothreads');
7078
7079     my $iothreads = {};
7080     foreach my $iothread (@$res) {
7081         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7082     }
7083
7084     return $iothreads;
7085 }
7086
7087 sub scsihw_infos {
7088     my ($conf, $drive) = @_;
7089
7090     my $maxdev = 0;
7091
7092     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7093         $maxdev = 7;
7094     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7095         $maxdev = 1;
7096     } else {
7097         $maxdev = 256;
7098     }
7099
7100     my $controller = int($drive->{index} / $maxdev);
7101     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7102         ? "virtioscsi"
7103         : "scsihw";
7104
7105     return ($maxdev, $controller, $controller_prefix);
7106 }
7107
7108 sub windows_version {
7109     my ($ostype) = @_;
7110
7111     return 0 if !$ostype;
7112
7113     my $winversion = 0;
7114
7115     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7116         $winversion = 5;
7117     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7118         $winversion = 6;
7119     } elsif ($ostype =~ m/^win(\d+)$/) {
7120         $winversion = $1;
7121     }
7122
7123     return $winversion;
7124 }
7125
7126 sub resolve_dst_disk_format {
7127         my ($storecfg, $storeid, $src_volname, $format) = @_;
7128         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7129
7130         if (!$format) {
7131             # if no target format is specified, use the source disk format as hint
7132             if ($src_volname) {
7133                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7134                 $format = qemu_img_format($scfg, $src_volname);
7135             } else {
7136                 return $defFormat;
7137             }
7138         }
7139
7140         # test if requested format is supported - else use default
7141         my $supported = grep { $_ eq $format } @$validFormats;
7142         $format = $defFormat if !$supported;
7143         return $format;
7144 }
7145
7146 # NOTE: if this logic changes, please update docs & possibly gui logic
7147 sub find_vmstate_storage {
7148     my ($conf, $storecfg) = @_;
7149
7150     # first, return storage from conf if set
7151     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7152
7153     my ($target, $shared, $local);
7154
7155     foreach_storage_used_by_vm($conf, sub {
7156         my ($sid) = @_;
7157         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7158         my $dst = $scfg->{shared} ? \$shared : \$local;
7159         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7160     });
7161
7162     # second, use shared storage where VM has at least one disk
7163     # third, use local storage where VM has at least one disk
7164     # fall back to local storage
7165     $target = $shared // $local // 'local';
7166
7167     return $target;
7168 }
7169
7170 sub generate_uuid {
7171     my ($uuid, $uuid_str);
7172     UUID::generate($uuid);
7173     UUID::unparse($uuid, $uuid_str);
7174     return $uuid_str;
7175 }
7176
7177 sub generate_smbios1_uuid {
7178     return "uuid=".generate_uuid();
7179 }
7180
7181 sub nbd_stop {
7182     my ($vmid) = @_;
7183
7184     mon_cmd($vmid, 'nbd-server-stop');
7185 }
7186
7187 sub create_reboot_request {
7188     my ($vmid) = @_;
7189     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7190         or die "failed to create reboot trigger file: $!\n";
7191     close($fh);
7192 }
7193
7194 sub clear_reboot_request {
7195     my ($vmid) = @_;
7196     my $path = "/run/qemu-server/$vmid.reboot";
7197     my $res = 0;
7198
7199     $res = unlink($path);
7200     die "could not remove reboot request for $vmid: $!"
7201         if !$res && $! != POSIX::ENOENT;
7202
7203     return $res;
7204 }
7205
7206 sub bootorder_from_legacy {
7207     my ($conf, $bootcfg) = @_;
7208
7209     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7210     my $bootindex_hash = {};
7211     my $i = 1;
7212     foreach my $o (split(//, $boot)) {
7213         $bootindex_hash->{$o} = $i*100;
7214         $i++;
7215     }
7216
7217     my $bootorder = {};
7218
7219     PVE::QemuConfig->foreach_volume($conf, sub {
7220         my ($ds, $drive) = @_;
7221
7222         if (drive_is_cdrom ($drive, 1)) {
7223             if ($bootindex_hash->{d}) {
7224                 $bootorder->{$ds} = $bootindex_hash->{d};
7225                 $bootindex_hash->{d} += 1;
7226             }
7227         } elsif ($bootindex_hash->{c}) {
7228             $bootorder->{$ds} = $bootindex_hash->{c}
7229                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7230             $bootindex_hash->{c} += 1;
7231         }
7232     });
7233
7234     if ($bootindex_hash->{n}) {
7235         for (my $i = 0; $i < $MAX_NETS; $i++) {
7236             my $netname = "net$i";
7237             next if !$conf->{$netname};
7238             $bootorder->{$netname} = $bootindex_hash->{n};
7239             $bootindex_hash->{n} += 1;
7240         }
7241     }
7242
7243     return $bootorder;
7244 }
7245
7246 # Generate default device list for 'boot: order=' property. Matches legacy
7247 # default boot order, but with explicit device names. This is important, since
7248 # the fallback for when neither 'order' nor the old format is specified relies
7249 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7250 sub get_default_bootdevices {
7251     my ($conf) = @_;
7252
7253     my @ret = ();
7254
7255     # harddisk
7256     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7257     push @ret, $first if $first;
7258
7259     # cdrom
7260     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7261     push @ret, $first if $first;
7262
7263     # network
7264     for (my $i = 0; $i < $MAX_NETS; $i++) {
7265         my $netname = "net$i";
7266         next if !$conf->{$netname};
7267         push @ret, $netname;
7268         last;
7269     }
7270
7271     return \@ret;
7272 }
7273
7274 sub device_bootorder {
7275     my ($conf) = @_;
7276
7277     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7278
7279     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7280
7281     my $bootorder = {};
7282     if (!defined($boot) || $boot->{legacy}) {
7283         $bootorder = bootorder_from_legacy($conf, $boot);
7284     } elsif ($boot->{order}) {
7285         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7286         for my $dev (PVE::Tools::split_list($boot->{order})) {
7287             $bootorder->{$dev} = $i++;
7288         }
7289     }
7290
7291     return $bootorder;
7292 }
7293
7294 # bash completion helper
7295
7296 sub complete_backup_archives {
7297     my ($cmdname, $pname, $cvalue) = @_;
7298
7299     my $cfg = PVE::Storage::config();
7300
7301     my $storeid;
7302
7303     if ($cvalue =~ m/^([^:]+):/) {
7304         $storeid = $1;
7305     }
7306
7307     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7308
7309     my $res = [];
7310     foreach my $id (keys %$data) {
7311         foreach my $item (@{$data->{$id}}) {
7312             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7313             push @$res, $item->{volid} if defined($item->{volid});
7314         }
7315     }
7316
7317     return $res;
7318 }
7319
7320 my $complete_vmid_full = sub {
7321     my ($running) = @_;
7322
7323     my $idlist = vmstatus();
7324
7325     my $res = [];
7326
7327     foreach my $id (keys %$idlist) {
7328         my $d = $idlist->{$id};
7329         if (defined($running)) {
7330             next if $d->{template};
7331             next if $running && $d->{status} ne 'running';
7332             next if !$running && $d->{status} eq 'running';
7333         }
7334         push @$res, $id;
7335
7336     }
7337     return $res;
7338 };
7339
7340 sub complete_vmid {
7341     return &$complete_vmid_full();
7342 }
7343
7344 sub complete_vmid_stopped {
7345     return &$complete_vmid_full(0);
7346 }
7347
7348 sub complete_vmid_running {
7349     return &$complete_vmid_full(1);
7350 }
7351
7352 sub complete_storage {
7353
7354     my $cfg = PVE::Storage::config();
7355     my $ids = $cfg->{ids};
7356
7357     my $res = [];
7358     foreach my $sid (keys %$ids) {
7359         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7360         next if !$ids->{$sid}->{content}->{images};
7361         push @$res, $sid;
7362     }
7363
7364     return $res;
7365 }
7366
7367 sub complete_migration_storage {
7368     my ($cmd, $param, $current_value, $all_args) = @_;
7369
7370     my $targetnode = @$all_args[1];
7371
7372     my $cfg = PVE::Storage::config();
7373     my $ids = $cfg->{ids};
7374
7375     my $res = [];
7376     foreach my $sid (keys %$ids) {
7377         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7378         next if !$ids->{$sid}->{content}->{images};
7379         push @$res, $sid;
7380     }
7381
7382     return $res;
7383 }
7384
7385 1;