PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016/2019
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud', 'opennebula'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 968
 969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 970 dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $usb_fmt = {
1002     host => {
1003         default_key => 1,
1004         type => 'string', format => 'pve-qm-usb-device',
1005         format_description => 'HOSTUSBDEVICE|spice',
1006         description => <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1008
1009  'bus-port(.port)*' (decimal numbers) or
1010  'vendor_id:product_id' (hexadeciaml numbers) or
1011  'spice'
1012
1013 You can use the 'lsusb -t' command to list existing usb devices.
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1017
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1019 EODESCR
1020     },
1021     usb3 => {
1022         optional => 1,
1023         type => 'boolean',
1024         description => "Specifies whether if given host option is a USB3 device or port.",
1025         default => 0,
1026     },
1027 };
1028
1029 my $usbdesc = {
1030     optional => 1,
1031     type => 'string', format => $usb_fmt,
1032     description => "Configure an USB device (n is 0 to 4).",
1033 };
1034 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1035
1036 my $serialdesc = {
1037         optional => 1,
1038         type => 'string',
1039         pattern => '(/dev/.+|socket)',
1040         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1041         verbose_description =>  <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1045
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1048
1049 CAUTION: Experimental! User reported problems with this option.
1050 EODESCR
1051 };
1052
1053 my $paralleldesc= {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1057         description =>  "Map host parallel devices (n is 0 to 2).",
1058         verbose_description =>  <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1060
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1069     $confdesc->{"parallel$i"} = $paralleldesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1073     $confdesc->{"serial$i"} = $serialdesc;
1074 }
1075
1076 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1077     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1078 }
1079
1080 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1081     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1082 }
1083
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1085     $confdesc->{"usb$i"} = $usbdesc;
1086 }
1087
1088 my $boot_fmt = {
1089     legacy => {
1090         optional => 1,
1091         default_key => 1,
1092         type => 'string',
1093         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094                      . " Deprecated, use 'order=' instead.",
1095         pattern => '[acdn]{1,4}',
1096         format_description => "[acdn]{1,4}",
1097
1098         # note: this is also the fallback if boot: is not given at all
1099         default => 'cdn',
1100     },
1101     order => {
1102         optional => 1,
1103         type => 'string',
1104         format => 'pve-qm-bootdev-list',
1105         format_description => "device[;device...]",
1106         description => <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1108
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1112
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1116
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1118 EODESC
1119     },
1120 };
1121 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1122
1123 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1124 sub verify_bootdev {
1125     my ($dev, $noerr) = @_;
1126
1127     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1128
1129     my $check = sub {
1130         my ($base) = @_;
1131         return 0 if $dev !~ m/^$base\d+$/;
1132         return 0 if !$confdesc->{$dev};
1133         return 1;
1134     };
1135
1136     return $dev if $check->("net");
1137     return $dev if $check->("usb");
1138     return $dev if $check->("hostpci");
1139
1140     return if $noerr;
1141     die "invalid boot device '$dev'\n";
1142 }
1143
1144 sub print_bootorder {
1145     my ($devs) = @_;
1146     return "" if !@$devs;
1147     my $data = { order => join(';', @$devs) };
1148     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1149 }
1150
1151 my $kvm_api_version = 0;
1152
1153 sub kvm_version {
1154     return $kvm_api_version if $kvm_api_version;
1155
1156     open my $fh, '<', '/dev/kvm' or return;
1157
1158     # 0xae00 => KVM_GET_API_VERSION
1159     $kvm_api_version = ioctl($fh, 0xae00, 0);
1160     close($fh);
1161
1162     return $kvm_api_version;
1163 }
1164
1165 my $kvm_user_version = {};
1166 my $kvm_mtime = {};
1167
1168 sub kvm_user_version {
1169     my ($binary) = @_;
1170
1171     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1172     my $st = stat($binary);
1173
1174     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176         $cachedmtime == $st->mtime;
1177
1178     $kvm_user_version->{$binary} = 'unknown';
1179     $kvm_mtime->{$binary} = $st->mtime;
1180
1181     my $code = sub {
1182         my $line = shift;
1183         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184             $kvm_user_version->{$binary} = $2;
1185         }
1186     };
1187
1188     eval { run_command([$binary, '--version'], outfunc => $code); };
1189     warn $@ if $@;
1190
1191     return $kvm_user_version->{$binary};
1192
1193 }
1194 my sub extract_version {
1195     my ($machine_type, $version) = @_;
1196     $version = kvm_user_version() if !defined($version);
1197     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1198 }
1199
1200 sub kernel_has_vhost_net {
1201     return -c '/dev/vhost-net';
1202 }
1203
1204 sub option_exists {
1205     my $key = shift;
1206     return defined($confdesc->{$key});
1207 }
1208
1209 my $cdrom_path;
1210 sub get_cdrom_path {
1211
1212     return  $cdrom_path if $cdrom_path;
1213
1214     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1215     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1216     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1217 }
1218
1219 sub get_iso_path {
1220     my ($storecfg, $vmid, $cdrom) = @_;
1221
1222     if ($cdrom eq 'cdrom') {
1223         return get_cdrom_path();
1224     } elsif ($cdrom eq 'none') {
1225         return '';
1226     } elsif ($cdrom =~ m|^/|) {
1227         return $cdrom;
1228     } else {
1229         return PVE::Storage::path($storecfg, $cdrom);
1230     }
1231 }
1232
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id {
1235     my ($vmid, $file, $media) = @_;
1236
1237      if (!($file eq 'none' || $file eq 'cdrom' ||
1238           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1239
1240         return if $file =~ m|/|;
1241
1242         if ($media && $media eq 'cdrom') {
1243             $file = "local:iso/$file";
1244         } else {
1245             $file = "local:$vmid/$file";
1246         }
1247     }
1248
1249     return $file;
1250 }
1251
1252 sub verify_media_type {
1253     my ($opt, $vtype, $media) = @_;
1254
1255     return if !$media;
1256
1257     my $etype;
1258     if ($media eq 'disk') {
1259         $etype = 'images';
1260     } elsif ($media eq 'cdrom') {
1261         $etype = 'iso';
1262     } else {
1263         die "internal error";
1264     }
1265
1266     return if ($vtype eq $etype);
1267
1268     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1269 }
1270
1271 sub cleanup_drive_path {
1272     my ($opt, $storecfg, $drive) = @_;
1273
1274     # try to convert filesystem paths to volume IDs
1275
1276     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1277         ($drive->{file} !~ m|^/dev/.+|) &&
1278         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1279         ($drive->{file} !~ m/^\d+$/)) {
1280         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1281         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1282             if !$vtype;
1283         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1284         verify_media_type($opt, $vtype, $drive->{media});
1285         $drive->{file} = $volid;
1286     }
1287
1288     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1289 }
1290
1291 sub parse_hotplug_features {
1292     my ($data) = @_;
1293
1294     my $res = {};
1295
1296     return $res if $data eq '0';
1297
1298     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1299
1300     foreach my $feature (PVE::Tools::split_list($data)) {
1301         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1302             $res->{$1} = 1;
1303         } else {
1304             die "invalid hotplug feature '$feature'\n";
1305         }
1306     }
1307     return $res;
1308 }
1309
1310 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1311 sub pve_verify_hotplug_features {
1312     my ($value, $noerr) = @_;
1313
1314     return $value if parse_hotplug_features($value);
1315
1316     return if $noerr;
1317
1318     die "unable to parse hotplug option\n";
1319 }
1320
1321 sub scsi_inquiry {
1322     my($fh, $noerr) = @_;
1323
1324     my $SG_IO = 0x2285;
1325     my $SG_GET_VERSION_NUM = 0x2282;
1326
1327     my $versionbuf = "\x00" x 8;
1328     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1329     if (!$ret) {
1330         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1331         return;
1332     }
1333     my $version = unpack("I", $versionbuf);
1334     if ($version < 30000) {
1335         die "scsi generic interface too old\n"  if !$noerr;
1336         return;
1337     }
1338
1339     my $buf = "\x00" x 36;
1340     my $sensebuf = "\x00" x 8;
1341     my $cmd = pack("C x3 C x1", 0x12, 36);
1342
1343     # see /usr/include/scsi/sg.h
1344     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1345
1346     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347                       length($sensebuf), 0, length($buf), $buf,
1348                       $cmd, $sensebuf, 6000);
1349
1350     $ret = ioctl($fh, $SG_IO, $packet);
1351     if (!$ret) {
1352         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1353         return;
1354     }
1355
1356     my @res = unpack($sg_io_hdr_t, $packet);
1357     if ($res[17] || $res[18]) {
1358         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1359         return;
1360     }
1361
1362     my $res = {};
1363     (my $byte0, my $byte1, $res->{vendor},
1364      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1365
1366     $res->{removable} = $byte1 & 128 ? 1 : 0;
1367     $res->{type} = $byte0 & 31;
1368
1369     return $res;
1370 }
1371
1372 sub path_is_scsi {
1373     my ($path) = @_;
1374
1375     my $fh = IO::File->new("+<$path") || return;
1376     my $res = scsi_inquiry($fh, 1);
1377     close($fh);
1378
1379     return $res;
1380 }
1381
1382 sub print_tabletdevice_full {
1383     my ($conf, $arch) = @_;
1384
1385     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1386
1387     # we use uhci for old VMs because tablet driver was buggy in older qemu
1388     my $usbbus;
1389     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1390         $usbbus = 'ehci';
1391     } else {
1392         $usbbus = 'uhci';
1393     }
1394
1395     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1396 }
1397
1398 sub print_keyboarddevice_full {
1399     my ($conf, $arch, $machine) = @_;
1400
1401     return if $arch ne 'aarch64';
1402
1403     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1404 }
1405
1406 sub print_drivedevice_full {
1407     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1408
1409     my $device = '';
1410     my $maxdev = 0;
1411
1412     my $drive_id = "$drive->{interface}$drive->{index}";
1413     if ($drive->{interface} eq 'virtio') {
1414         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1415         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1417     } elsif ($drive->{interface} eq 'scsi') {
1418
1419         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1420         my $unit = $drive->{index} % $maxdev;
1421         my $devicetype = 'hd';
1422         my $path = '';
1423         if (drive_is_cdrom($drive)) {
1424             $devicetype = 'cd';
1425         } else {
1426             if ($drive->{file} =~ m|^/|) {
1427                 $path = $drive->{file};
1428                 if (my $info = path_is_scsi($path)) {
1429                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1430                         $devicetype = 'block';
1431                     } elsif ($info->{type} == 1) { # tape
1432                         $devicetype = 'generic';
1433                     }
1434                 }
1435             } else {
1436                  $path = PVE::Storage::path($storecfg, $drive->{file});
1437             }
1438
1439             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440             my $version = extract_version($machine_type, kvm_user_version());
1441             if ($path =~ m/^iscsi\:\/\// &&
1442                !min_version($version, 4, 1)) {
1443                 $devicetype = 'generic';
1444             }
1445         }
1446
1447         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1448             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1449         } else {
1450             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451                 .",lun=$drive->{index}";
1452         }
1453         $device .= ",drive=drive-$drive_id,id=$drive_id";
1454
1455         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456             $device .= ",rotation_rate=1";
1457         }
1458         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1459
1460     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1461         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1462         my $controller = int($drive->{index} / $maxdev);
1463         my $unit = $drive->{index} % $maxdev;
1464         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1465
1466         $device = "ide-$devicetype";
1467         if ($drive->{interface} eq 'ide') {
1468             $device .= ",bus=ide.$controller,unit=$unit";
1469         } else {
1470             $device .= ",bus=ahci$controller.$unit";
1471         }
1472         $device .= ",drive=drive-$drive_id,id=$drive_id";
1473
1474         if ($devicetype eq 'hd') {
1475             if (my $model = $drive->{model}) {
1476                 $model = URI::Escape::uri_unescape($model);
1477                 $device .= ",model=$model";
1478             }
1479             if ($drive->{ssd}) {
1480                 $device .= ",rotation_rate=1";
1481             }
1482         }
1483         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1484     } elsif ($drive->{interface} eq 'usb') {
1485         die "implement me";
1486         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1487     } else {
1488         die "unsupported interface type";
1489     }
1490
1491     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1492
1493     if (my $serial = $drive->{serial}) {
1494         $serial = URI::Escape::uri_unescape($serial);
1495         $device .= ",serial=$serial";
1496     }
1497
1498
1499     return $device;
1500 }
1501
1502 sub get_initiator_name {
1503     my $initiator;
1504
1505     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1506     while (defined(my $line = <$fh>)) {
1507         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1508         $initiator = $1;
1509         last;
1510     }
1511     $fh->close();
1512
1513     return $initiator;
1514 }
1515
1516 sub print_drive_commandline_full {
1517     my ($storecfg, $vmid, $drive) = @_;
1518
1519     my $path;
1520     my $volid = $drive->{file};
1521     my $format;
1522
1523     if (drive_is_cdrom($drive)) {
1524         $path = get_iso_path($storecfg, $vmid, $volid);
1525     } else {
1526         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1527         if ($storeid) {
1528             $path = PVE::Storage::path($storecfg, $volid);
1529             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1530             $format = qemu_img_format($scfg, $volname);
1531         } else {
1532             $path = $volid;
1533             $format = "raw";
1534         }
1535    }
1536
1537     my $opts = '';
1538     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539     foreach my $o (@qemu_drive_options) {
1540         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1541     }
1542
1543     # snapshot only accepts on|off
1544     if (defined($drive->{snapshot})) {
1545         my $v = $drive->{snapshot} ? 'on' : 'off';
1546         $opts .= ",snapshot=$v";
1547     }
1548
1549     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1550         my ($dir, $qmpname) = @$type;
1551         if (my $v = $drive->{"mbps$dir"}) {
1552             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1553         }
1554         if (my $v = $drive->{"mbps${dir}_max"}) {
1555             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1556         }
1557         if (my $v = $drive->{"bps${dir}_max_length"}) {
1558             $opts .= ",throttling.bps$qmpname-max-length=$v";
1559         }
1560         if (my $v = $drive->{"iops${dir}"}) {
1561             $opts .= ",throttling.iops$qmpname=$v";
1562         }
1563         if (my $v = $drive->{"iops${dir}_max"}) {
1564             $opts .= ",throttling.iops$qmpname-max=$v";
1565         }
1566         if (my $v = $drive->{"iops${dir}_max_length"}) {
1567             $opts .= ",throttling.iops$qmpname-max-length=$v";
1568         }
1569     }
1570
1571     $opts .= ",format=$format" if $format && !$drive->{format};
1572
1573     my $cache_direct = 0;
1574
1575     if (my $cache = $drive->{cache}) {
1576         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577     } elsif (!drive_is_cdrom($drive)) {
1578         $opts .= ",cache=none";
1579         $cache_direct = 1;
1580     }
1581
1582     # aio native works only with O_DIRECT
1583     if (!$drive->{aio}) {
1584         if($cache_direct) {
1585             $opts .= ",aio=native";
1586         } else {
1587             $opts .= ",aio=threads";
1588         }
1589     }
1590
1591     if (!drive_is_cdrom($drive)) {
1592         my $detectzeroes;
1593         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1594             $detectzeroes = 'off';
1595         } elsif ($drive->{discard}) {
1596             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1597         } else {
1598             # This used to be our default with discard not being specified:
1599             $detectzeroes = 'on';
1600         }
1601         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1602     }
1603
1604     my $pathinfo = $path ? "file=$path," : '';
1605
1606     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1607 }
1608
1609 sub print_netdevice_full {
1610     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1611
1612     my $device = $net->{model};
1613     if ($net->{model} eq 'virtio') {
1614          $device = 'virtio-net-pci';
1615      };
1616
1617     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1618     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1620         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621         # and out of each queue plus one config interrupt and control vector queue
1622         my $vectors = $net->{queues} * 2 + 2;
1623         $tmpstr .= ",vectors=$vectors,mq=on";
1624     }
1625     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1626
1627     if (my $mtu = $net->{mtu}) {
1628         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1629             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1630             if ($mtu == 1) {
1631                  $mtu = $bridge_mtu;
1632             } elsif ($mtu < 576) {
1633                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634             } elsif ($mtu > $bridge_mtu) {
1635                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1636             }
1637             $tmpstr .= ",host_mtu=$mtu";
1638         } else {
1639             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1640         }
1641     }
1642
1643     if ($use_old_bios_files) {
1644         my $romfile;
1645         if ($device eq 'virtio-net-pci') {
1646             $romfile = 'pxe-virtio.rom';
1647         } elsif ($device eq 'e1000') {
1648             $romfile = 'pxe-e1000.rom';
1649         } elsif ($device eq 'ne2k') {
1650             $romfile = 'pxe-ne2k_pci.rom';
1651         } elsif ($device eq 'pcnet') {
1652             $romfile = 'pxe-pcnet.rom';
1653         } elsif ($device eq 'rtl8139') {
1654             $romfile = 'pxe-rtl8139.rom';
1655         }
1656         $tmpstr .= ",romfile=$romfile" if $romfile;
1657     }
1658
1659     return $tmpstr;
1660 }
1661
1662 sub print_netdev_full {
1663     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1664
1665     my $i = '';
1666     if ($netid =~ m/^net(\d+)$/) {
1667         $i = int($1);
1668     }
1669
1670     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1671
1672     my $ifname = "tap${vmid}i$i";
1673
1674     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675     die "interface name '$ifname' is too long (max 15 character)\n"
1676         if length($ifname) >= 16;
1677
1678     my $vhostparam = '';
1679     if (is_native($arch)) {
1680         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1681     }
1682
1683     my $vmname = $conf->{name} || "vm$vmid";
1684
1685     my $netdev = "";
1686     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1687
1688     if ($net->{bridge}) {
1689         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1691     } else {
1692         $netdev = "type=user,id=$netid,hostname=$vmname";
1693     }
1694
1695     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1696
1697     return $netdev;
1698 }
1699
1700 my $vga_map = {
1701     'cirrus' => 'cirrus-vga',
1702     'std' => 'VGA',
1703     'vmware' => 'vmware-svga',
1704     'virtio' => 'virtio-vga',
1705 };
1706
1707 sub print_vga_device {
1708     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1709
1710     my $type = $vga_map->{$vga->{type}};
1711     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712         $type = 'virtio-gpu';
1713     }
1714     my $vgamem_mb = $vga->{memory};
1715
1716     my $max_outputs = '';
1717     if ($qxlnum) {
1718         $type = $id ? 'qxl' : 'qxl-vga';
1719
1720         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1721             # set max outputs so linux can have up to 4 qxl displays with one device
1722             if (min_version($machine_version, 4, 1)) {
1723                 $max_outputs = ",max_outputs=4";
1724             }
1725         }
1726     }
1727
1728     die "no devicetype for $vga->{type}\n" if !$type;
1729
1730     my $memory = "";
1731     if ($vgamem_mb) {
1732         if ($vga->{type} eq 'virtio') {
1733             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1734             $memory = ",max_hostmem=$bytes";
1735         } elsif ($qxlnum) {
1736             # from https://www.spice-space.org/multiple-monitors.html
1737             $memory = ",vgamem_mb=$vga->{memory}";
1738             my $ram = $vgamem_mb * 4;
1739             my $vram = $vgamem_mb * 2;
1740             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1741         } else {
1742             $memory = ",vgamem_mb=$vga->{memory}";
1743         }
1744     } elsif ($qxlnum && $id) {
1745         $memory = ",ram_size=67108864,vram_size=33554432";
1746     }
1747
1748     my $edidoff = "";
1749     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1750         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1751     }
1752
1753     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1754     my $vgaid = "vga" . ($id // '');
1755     my $pciaddr;
1756
1757     if ($q35 && $vgaid eq 'vga') {
1758         # the first display uses pcie.0 bus on q35 machines
1759         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1760     } else {
1761         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1762     }
1763
1764     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1765 }
1766
1767 sub parse_number_sets {
1768     my ($set) = @_;
1769     my $res = [];
1770     foreach my $part (split(/;/, $set)) {
1771         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773             push @$res, [ $1, $2 ];
1774         } else {
1775             die "invalid range: $part\n";
1776         }
1777     }
1778     return $res;
1779 }
1780
1781 sub parse_numa {
1782     my ($data) = @_;
1783
1784     my $res = parse_property_string($numa_fmt, $data);
1785     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1786     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1787     return $res;
1788 }
1789
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1791 sub parse_net {
1792     my ($data) = @_;
1793
1794     my $res = eval { parse_property_string($net_fmt, $data) };
1795     if ($@) {
1796         warn $@;
1797         return;
1798     }
1799     if (!defined($res->{macaddr})) {
1800         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1801         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1802     }
1803     return $res;
1804 }
1805
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig {
1808     my ($data) = @_;
1809
1810     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1811     if ($@) {
1812         warn $@;
1813         return;
1814     }
1815
1816     if ($res->{gw} && !$res->{ip}) {
1817         warn 'gateway specified without specifying an IP address';
1818         return;
1819     }
1820     if ($res->{gw6} && !$res->{ip6}) {
1821         warn 'IPv6 gateway specified without specifying an IPv6 address';
1822         return;
1823     }
1824     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1825         warn 'gateway specified together with DHCP';
1826         return;
1827     }
1828     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1829         # gw6 + auto/dhcp
1830         warn "IPv6 gateway specified together with $res->{ip6} address";
1831         return;
1832     }
1833
1834     if (!$res->{ip} && !$res->{ip6}) {
1835         return { ip => 'dhcp', ip6 => 'dhcp' };
1836     }
1837
1838     return $res;
1839 }
1840
1841 sub print_net {
1842     my $net = shift;
1843
1844     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1845 }
1846
1847 sub add_random_macs {
1848     my ($settings) = @_;
1849
1850     foreach my $opt (keys %$settings) {
1851         next if $opt !~ m/^net(\d+)$/;
1852         my $net = parse_net($settings->{$opt});
1853         next if !$net;
1854         $settings->{$opt} = print_net($net);
1855     }
1856 }
1857
1858 sub vm_is_volid_owner {
1859     my ($storecfg, $vmid, $volid) = @_;
1860
1861     if ($volid !~  m|^/|) {
1862         my ($path, $owner);
1863         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1864         if ($owner && ($owner == $vmid)) {
1865             return 1;
1866         }
1867     }
1868
1869     return;
1870 }
1871
1872 sub vmconfig_register_unused_drive {
1873     my ($storecfg, $vmid, $conf, $drive) = @_;
1874
1875     if (drive_is_cloudinit($drive)) {
1876         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1877         warn $@ if $@;
1878     } elsif (!drive_is_cdrom($drive)) {
1879         my $volid = $drive->{file};
1880         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1881             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1882         }
1883     }
1884 }
1885
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1887 my $smbios1_fmt = {
1888     uuid => {
1889         type => 'string',
1890         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891         format_description => 'UUID',
1892         description => "Set SMBIOS1 UUID.",
1893         optional => 1,
1894     },
1895     version => {
1896         type => 'string',
1897         pattern => '[A-Za-z0-9+\/]+={0,2}',
1898         format_description => 'Base64 encoded string',
1899         description => "Set SMBIOS1 version.",
1900         optional => 1,
1901     },
1902     serial => {
1903         type => 'string',
1904         pattern => '[A-Za-z0-9+\/]+={0,2}',
1905         format_description => 'Base64 encoded string',
1906         description => "Set SMBIOS1 serial number.",
1907         optional => 1,
1908     },
1909     manufacturer => {
1910         type => 'string',
1911         pattern => '[A-Za-z0-9+\/]+={0,2}',
1912         format_description => 'Base64 encoded string',
1913         description => "Set SMBIOS1 manufacturer.",
1914         optional => 1,
1915     },
1916     product => {
1917         type => 'string',
1918         pattern => '[A-Za-z0-9+\/]+={0,2}',
1919         format_description => 'Base64 encoded string',
1920         description => "Set SMBIOS1 product ID.",
1921         optional => 1,
1922     },
1923     sku => {
1924         type => 'string',
1925         pattern => '[A-Za-z0-9+\/]+={0,2}',
1926         format_description => 'Base64 encoded string',
1927         description => "Set SMBIOS1 SKU string.",
1928         optional => 1,
1929     },
1930     family => {
1931         type => 'string',
1932         pattern => '[A-Za-z0-9+\/]+={0,2}',
1933         format_description => 'Base64 encoded string',
1934         description => "Set SMBIOS1 family string.",
1935         optional => 1,
1936     },
1937     base64 => {
1938         type => 'boolean',
1939         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1940         optional => 1,
1941     },
1942 };
1943
1944 sub parse_smbios1 {
1945     my ($data) = @_;
1946
1947     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1948     warn $@ if $@;
1949     return $res;
1950 }
1951
1952 sub print_smbios1 {
1953     my ($smbios1) = @_;
1954     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1955 }
1956
1957 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1958
1959 sub parse_watchdog {
1960     my ($value) = @_;
1961
1962     return if !$value;
1963
1964     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1965     warn $@ if $@;
1966     return $res;
1967 }
1968
1969 sub parse_guest_agent {
1970     my ($conf) = @_;
1971
1972     return {} if !defined($conf->{agent});
1973
1974     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
1975     warn $@ if $@;
1976
1977     # if the agent is disabled ignore the other potentially set properties
1978     return {} if !$res->{enabled};
1979     return $res;
1980 }
1981
1982 sub get_qga_key {
1983     my ($conf, $key) = @_;
1984     return undef if !defined($conf->{agent});
1985
1986     my $agent = parse_guest_agent($conf);
1987     return $agent->{$key};
1988 }
1989
1990 sub parse_vga {
1991     my ($value) = @_;
1992
1993     return {} if !$value;
1994     my $res = eval { parse_property_string($vga_fmt, $value) };
1995     warn $@ if $@;
1996     return $res;
1997 }
1998
1999 sub parse_rng {
2000     my ($value) = @_;
2001
2002     return if !$value;
2003
2004     my $res = eval { parse_property_string($rng_fmt, $value) };
2005     warn $@ if $@;
2006     return $res;
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2010 sub verify_usb_device {
2011     my ($value, $noerr) = @_;
2012
2013     return $value if parse_usb_device($value);
2014
2015     return if $noerr;
2016
2017     die "unable to parse usb device\n";
2018 }
2019
2020 # add JSON properties for create and set function
2021 sub json_config_properties {
2022     my $prop = shift;
2023
2024     foreach my $opt (keys %$confdesc) {
2025         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027         $prop->{$opt} = $confdesc->{$opt};
2028     }
2029
2030     return $prop;
2031 }
2032
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties {
2035
2036     return dclone($confdesc_cloudinit);
2037 }
2038
2039 sub check_type {
2040     my ($key, $value) = @_;
2041
2042     die "unknown setting '$key'\n" if !$confdesc->{$key};
2043
2044     my $type = $confdesc->{$key}->{type};
2045
2046     if (!defined($value)) {
2047         die "got undefined value\n";
2048     }
2049
2050     if ($value =~ m/[\n\r]/) {
2051         die "property contains a line feed\n";
2052     }
2053
2054     if ($type eq 'boolean') {
2055         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057         die "type check ('boolean') failed - got '$value'\n";
2058     } elsif ($type eq 'integer') {
2059         return int($1) if $value =~ m/^(\d+)$/;
2060         die "type check ('integer') failed - got '$value'\n";
2061     } elsif ($type eq 'number') {
2062         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063         die "type check ('number') failed - got '$value'\n";
2064     } elsif ($type eq 'string') {
2065         if (my $fmt = $confdesc->{$key}->{format}) {
2066             PVE::JSONSchema::check_format($fmt, $value);
2067             return $value;
2068         }
2069         $value =~ s/^\"(.*)\"$/$1/;
2070         return $value;
2071     } else {
2072         die "internal error"
2073     }
2074 }
2075
2076 sub destroy_vm {
2077     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2078
2079     my $conf = PVE::QemuConfig->load_config($vmid);
2080
2081     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2082
2083     if ($conf->{template}) {
2084         # check if any base image is still used by a linked clone
2085         PVE::QemuConfig->foreach_volume($conf, sub {
2086                 my ($ds, $drive) = @_;
2087                 return if drive_is_cdrom($drive);
2088
2089                 my $volid = $drive->{file};
2090                 return if !$volid || $volid =~ m|^/|;
2091
2092                 die "base volume '$volid' is still in use by linked cloned\n"
2093                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2094
2095         });
2096     }
2097
2098     # only remove disks owned by this VM (referenced in the config)
2099     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2100         my ($ds, $drive) = @_;
2101         return if drive_is_cdrom($drive, 1);
2102
2103         my $volid = $drive->{file};
2104         return if !$volid || $volid =~ m|^/|;
2105
2106         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2107         return if !$path || !$owner || ($owner != $vmid);
2108
2109         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2110         warn "Could not remove disk '$volid', check manually: $@" if $@;
2111     });
2112
2113     if ($purge_unreferenced) { # also remove unreferenced disk
2114         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2115         PVE::Storage::foreach_volid($vmdisks, sub {
2116             my ($volid, $sid, $volname, $d) = @_;
2117             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2118             warn $@ if $@;
2119         });
2120     }
2121
2122     if (defined $replacement_conf) {
2123         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2124     } else {
2125         PVE::QemuConfig->destroy_config($vmid);
2126     }
2127 }
2128
2129 sub parse_vm_config {
2130     my ($filename, $raw) = @_;
2131
2132     return if !defined($raw);
2133
2134     my $res = {
2135         digest => Digest::SHA::sha1_hex($raw),
2136         snapshots => {},
2137         pending => {},
2138     };
2139
2140     $filename =~ m|/qemu-server/(\d+)\.conf$|
2141         || die "got strange filename '$filename'";
2142
2143     my $vmid = $1;
2144
2145     my $conf = $res;
2146     my $descr;
2147     my $section = '';
2148
2149     my @lines = split(/\n/, $raw);
2150     foreach my $line (@lines) {
2151         next if $line =~ m/^\s*$/;
2152
2153         if ($line =~ m/^\[PENDING\]\s*$/i) {
2154             $section = 'pending';
2155             if (defined($descr)) {
2156                 $descr =~ s/\s+$//;
2157                 $conf->{description} = $descr;
2158             }
2159             $descr = undef;
2160             $conf = $res->{$section} = {};
2161             next;
2162
2163         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2164             $section = $1;
2165             if (defined($descr)) {
2166                 $descr =~ s/\s+$//;
2167                 $conf->{description} = $descr;
2168             }
2169             $descr = undef;
2170             $conf = $res->{snapshots}->{$section} = {};
2171             next;
2172         }
2173
2174         if ($line =~ m/^\#(.*)\s*$/) {
2175             $descr = '' if !defined($descr);
2176             $descr .= PVE::Tools::decode_text($1) . "\n";
2177             next;
2178         }
2179
2180         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2181             $descr = '' if !defined($descr);
2182             $descr .= PVE::Tools::decode_text($2);
2183         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2184             $conf->{snapstate} = $1;
2185         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2186             my $key = $1;
2187             my $value = $2;
2188             $conf->{$key} = $value;
2189         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2190             my $value = $1;
2191             if ($section eq 'pending') {
2192                 $conf->{delete} = $value; # we parse this later
2193             } else {
2194                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2195             }
2196         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2197             my $key = $1;
2198             my $value = $2;
2199             eval { $value = check_type($key, $value); };
2200             if ($@) {
2201                 warn "vm $vmid - unable to parse value of '$key' - $@";
2202             } else {
2203                 $key = 'ide2' if $key eq 'cdrom';
2204                 my $fmt = $confdesc->{$key}->{format};
2205                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2206                     my $v = parse_drive($key, $value);
2207                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2208                         $v->{file} = $volid;
2209                         $value = print_drive($v);
2210                     } else {
2211                         warn "vm $vmid - unable to parse value of '$key'\n";
2212                         next;
2213                     }
2214                 }
2215
2216                 $conf->{$key} = $value;
2217             }
2218         } else {
2219             warn "vm $vmid - unable to parse config: $line\n";
2220         }
2221     }
2222
2223     if (defined($descr)) {
2224         $descr =~ s/\s+$//;
2225         $conf->{description} = $descr;
2226     }
2227     delete $res->{snapstate}; # just to be sure
2228
2229     return $res;
2230 }
2231
2232 sub write_vm_config {
2233     my ($filename, $conf) = @_;
2234
2235     delete $conf->{snapstate}; # just to be sure
2236
2237     if ($conf->{cdrom}) {
2238         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2239         $conf->{ide2} = $conf->{cdrom};
2240         delete $conf->{cdrom};
2241     }
2242
2243     # we do not use 'smp' any longer
2244     if ($conf->{sockets}) {
2245         delete $conf->{smp};
2246     } elsif ($conf->{smp}) {
2247         $conf->{sockets} = $conf->{smp};
2248         delete $conf->{cores};
2249         delete $conf->{smp};
2250     }
2251
2252     my $used_volids = {};
2253
2254     my $cleanup_config = sub {
2255         my ($cref, $pending, $snapname) = @_;
2256
2257         foreach my $key (keys %$cref) {
2258             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2259                 $key eq 'snapstate' || $key eq 'pending';
2260             my $value = $cref->{$key};
2261             if ($key eq 'delete') {
2262                 die "propertry 'delete' is only allowed in [PENDING]\n"
2263                     if !$pending;
2264                 # fixme: check syntax?
2265                 next;
2266             }
2267             eval { $value = check_type($key, $value); };
2268             die "unable to parse value of '$key' - $@" if $@;
2269
2270             $cref->{$key} = $value;
2271
2272             if (!$snapname && is_valid_drivename($key)) {
2273                 my $drive = parse_drive($key, $value);
2274                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2275             }
2276         }
2277     };
2278
2279     &$cleanup_config($conf);
2280
2281     &$cleanup_config($conf->{pending}, 1);
2282
2283     foreach my $snapname (keys %{$conf->{snapshots}}) {
2284         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2285         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2286     }
2287
2288     # remove 'unusedX' settings if we re-add a volume
2289     foreach my $key (keys %$conf) {
2290         my $value = $conf->{$key};
2291         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2292             delete $conf->{$key};
2293         }
2294     }
2295
2296     my $generate_raw_config = sub {
2297         my ($conf, $pending) = @_;
2298
2299         my $raw = '';
2300
2301         # add description as comment to top of file
2302         if (defined(my $descr = $conf->{description})) {
2303             if ($descr) {
2304                 foreach my $cl (split(/\n/, $descr)) {
2305                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2306                 }
2307             } else {
2308                 $raw .= "#\n" if $pending;
2309             }
2310         }
2311
2312         foreach my $key (sort keys %$conf) {
2313             next if $key =~ /^(digest|description|pending|snapshots)$/;
2314             $raw .= "$key: $conf->{$key}\n";
2315         }
2316         return $raw;
2317     };
2318
2319     my $raw = &$generate_raw_config($conf);
2320
2321     if (scalar(keys %{$conf->{pending}})){
2322         $raw .= "\n[PENDING]\n";
2323         $raw .= &$generate_raw_config($conf->{pending}, 1);
2324     }
2325
2326     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2327         $raw .= "\n[$snapname]\n";
2328         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2329     }
2330
2331     return $raw;
2332 }
2333
2334 sub load_defaults {
2335
2336     my $res = {};
2337
2338     # we use static defaults from our JSON schema configuration
2339     foreach my $key (keys %$confdesc) {
2340         if (defined(my $default = $confdesc->{$key}->{default})) {
2341             $res->{$key} = $default;
2342         }
2343     }
2344
2345     return $res;
2346 }
2347
2348 sub config_list {
2349     my $vmlist = PVE::Cluster::get_vmlist();
2350     my $res = {};
2351     return $res if !$vmlist || !$vmlist->{ids};
2352     my $ids = $vmlist->{ids};
2353     my $nodename = nodename();
2354
2355     foreach my $vmid (keys %$ids) {
2356         my $d = $ids->{$vmid};
2357         next if !$d->{node} || $d->{node} ne $nodename;
2358         next if !$d->{type} || $d->{type} ne 'qemu';
2359         $res->{$vmid}->{exists} = 1;
2360     }
2361     return $res;
2362 }
2363
2364 # test if VM uses local resources (to prevent migration)
2365 sub check_local_resources {
2366     my ($conf, $noerr) = @_;
2367
2368     my @loc_res = ();
2369
2370     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2371     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2372
2373     push @loc_res, "ivshmem" if $conf->{ivshmem};
2374
2375     foreach my $k (keys %$conf) {
2376         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2377         # sockets are safe: they will recreated be on the target side post-migrate
2378         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2379         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2380     }
2381
2382     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2383
2384     return \@loc_res;
2385 }
2386
2387 # check if used storages are available on all nodes (use by migrate)
2388 sub check_storage_availability {
2389     my ($storecfg, $conf, $node) = @_;
2390
2391     PVE::QemuConfig->foreach_volume($conf, sub {
2392         my ($ds, $drive) = @_;
2393
2394         my $volid = $drive->{file};
2395         return if !$volid;
2396
2397         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2398         return if !$sid;
2399
2400         # check if storage is available on both nodes
2401         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2402         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2403    });
2404 }
2405
2406 # list nodes where all VM images are available (used by has_feature API)
2407 sub shared_nodes {
2408     my ($conf, $storecfg) = @_;
2409
2410     my $nodelist = PVE::Cluster::get_nodelist();
2411     my $nodehash = { map { $_ => 1 } @$nodelist };
2412     my $nodename = nodename();
2413
2414     PVE::QemuConfig->foreach_volume($conf, sub {
2415         my ($ds, $drive) = @_;
2416
2417         my $volid = $drive->{file};
2418         return if !$volid;
2419
2420         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2421         if ($storeid) {
2422             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2423             if ($scfg->{disable}) {
2424                 $nodehash = {};
2425             } elsif (my $avail = $scfg->{nodes}) {
2426                 foreach my $node (keys %$nodehash) {
2427                     delete $nodehash->{$node} if !$avail->{$node};
2428                 }
2429             } elsif (!$scfg->{shared}) {
2430                 foreach my $node (keys %$nodehash) {
2431                     delete $nodehash->{$node} if $node ne $nodename
2432                 }
2433             }
2434         }
2435     });
2436
2437     return $nodehash
2438 }
2439
2440 sub check_local_storage_availability {
2441     my ($conf, $storecfg) = @_;
2442
2443     my $nodelist = PVE::Cluster::get_nodelist();
2444     my $nodehash = { map { $_ => {} } @$nodelist };
2445
2446     PVE::QemuConfig->foreach_volume($conf, sub {
2447         my ($ds, $drive) = @_;
2448
2449         my $volid = $drive->{file};
2450         return if !$volid;
2451
2452         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2453         if ($storeid) {
2454             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2455
2456             if ($scfg->{disable}) {
2457                 foreach my $node (keys %$nodehash) {
2458                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2459                 }
2460             } elsif (my $avail = $scfg->{nodes}) {
2461                 foreach my $node (keys %$nodehash) {
2462                     if (!$avail->{$node}) {
2463                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2464                     }
2465                 }
2466             }
2467         }
2468     });
2469
2470     foreach my $node (values %$nodehash) {
2471         if (my $unavail = $node->{unavailable_storages}) {
2472             $node->{unavailable_storages} = [ sort keys %$unavail ];
2473         }
2474     }
2475
2476     return $nodehash
2477 }
2478
2479 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2480 sub check_running {
2481     my ($vmid, $nocheck, $node) = @_;
2482
2483     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2484     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2485 }
2486
2487 sub vzlist {
2488
2489     my $vzlist = config_list();
2490
2491     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2492
2493     while (defined(my $de = $fd->read)) {
2494         next if $de !~ m/^(\d+)\.pid$/;
2495         my $vmid = $1;
2496         next if !defined($vzlist->{$vmid});
2497         if (my $pid = check_running($vmid)) {
2498             $vzlist->{$vmid}->{pid} = $pid;
2499         }
2500     }
2501
2502     return $vzlist;
2503 }
2504
2505 our $vmstatus_return_properties = {
2506     vmid => get_standard_option('pve-vmid'),
2507     status => {
2508         description => "Qemu process status.",
2509         type => 'string',
2510         enum => ['stopped', 'running'],
2511     },
2512     maxmem => {
2513         description => "Maximum memory in bytes.",
2514         type => 'integer',
2515         optional => 1,
2516         renderer => 'bytes',
2517     },
2518     maxdisk => {
2519         description => "Root disk size in bytes.",
2520         type => 'integer',
2521         optional => 1,
2522         renderer => 'bytes',
2523     },
2524     name => {
2525         description => "VM name.",
2526         type => 'string',
2527         optional => 1,
2528     },
2529     qmpstatus => {
2530         description => "Qemu QMP agent status.",
2531         type => 'string',
2532         optional => 1,
2533     },
2534     pid => {
2535         description => "PID of running qemu process.",
2536         type => 'integer',
2537         optional => 1,
2538     },
2539     uptime => {
2540         description => "Uptime.",
2541         type => 'integer',
2542         optional => 1,
2543         renderer => 'duration',
2544     },
2545     cpus => {
2546         description => "Maximum usable CPUs.",
2547         type => 'number',
2548         optional => 1,
2549     },
2550     lock => {
2551         description => "The current config lock, if any.",
2552         type => 'string',
2553         optional => 1,
2554     },
2555     tags => {
2556         description  => "The current configured tags, if any",
2557         type => 'string',
2558         optional => 1,
2559     },
2560     'running-machine' => {
2561         description => "The currently running machine type (if running).",
2562         type => 'string',
2563         optional => 1,
2564     },
2565     'running-qemu' => {
2566         description => "The currently running QEMU version (if running).",
2567         type => 'string',
2568         optional => 1,
2569     },
2570 };
2571
2572 my $last_proc_pid_stat;
2573
2574 # get VM status information
2575 # This must be fast and should not block ($full == false)
2576 # We only query KVM using QMP if $full == true (this can be slow)
2577 sub vmstatus {
2578     my ($opt_vmid, $full) = @_;
2579
2580     my $res = {};
2581
2582     my $storecfg = PVE::Storage::config();
2583
2584     my $list = vzlist();
2585     my $defaults = load_defaults();
2586
2587     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2588
2589     my $cpucount = $cpuinfo->{cpus} || 1;
2590
2591     foreach my $vmid (keys %$list) {
2592         next if $opt_vmid && ($vmid ne $opt_vmid);
2593
2594         my $conf = PVE::QemuConfig->load_config($vmid);
2595
2596         my $d = { vmid => $vmid };
2597         $d->{pid} = $list->{$vmid}->{pid};
2598
2599         # fixme: better status?
2600         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2601
2602         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2603         if (defined($size)) {
2604             $d->{disk} = 0; # no info available
2605             $d->{maxdisk} = $size;
2606         } else {
2607             $d->{disk} = 0;
2608             $d->{maxdisk} = 0;
2609         }
2610
2611         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2612             * ($conf->{cores} || $defaults->{cores});
2613         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2614         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2615
2616         $d->{name} = $conf->{name} || "VM $vmid";
2617         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2618             : $defaults->{memory}*(1024*1024);
2619
2620         if ($conf->{balloon}) {
2621             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2622             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2623                 : $defaults->{shares};
2624         }
2625
2626         $d->{uptime} = 0;
2627         $d->{cpu} = 0;
2628         $d->{mem} = 0;
2629
2630         $d->{netout} = 0;
2631         $d->{netin} = 0;
2632
2633         $d->{diskread} = 0;
2634         $d->{diskwrite} = 0;
2635
2636         $d->{template} = PVE::QemuConfig->is_template($conf);
2637
2638         $d->{serial} = 1 if conf_has_serial($conf);
2639         $d->{lock} = $conf->{lock} if $conf->{lock};
2640         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2641
2642         $res->{$vmid} = $d;
2643     }
2644
2645     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2646     foreach my $dev (keys %$netdev) {
2647         next if $dev !~ m/^tap([1-9]\d*)i/;
2648         my $vmid = $1;
2649         my $d = $res->{$vmid};
2650         next if !$d;
2651
2652         $d->{netout} += $netdev->{$dev}->{receive};
2653         $d->{netin} += $netdev->{$dev}->{transmit};
2654
2655         if ($full) {
2656             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2657             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2658         }
2659
2660     }
2661
2662     my $ctime = gettimeofday;
2663
2664     foreach my $vmid (keys %$list) {
2665
2666         my $d = $res->{$vmid};
2667         my $pid = $d->{pid};
2668         next if !$pid;
2669
2670         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2671         next if !$pstat; # not running
2672
2673         my $used = $pstat->{utime} + $pstat->{stime};
2674
2675         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2676
2677         if ($pstat->{vsize}) {
2678             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2679         }
2680
2681         my $old = $last_proc_pid_stat->{$pid};
2682         if (!$old) {
2683             $last_proc_pid_stat->{$pid} = {
2684                 time => $ctime,
2685                 used => $used,
2686                 cpu => 0,
2687             };
2688             next;
2689         }
2690
2691         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2692
2693         if ($dtime > 1000) {
2694             my $dutime = $used -  $old->{used};
2695
2696             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2697             $last_proc_pid_stat->{$pid} = {
2698                 time => $ctime,
2699                 used => $used,
2700                 cpu => $d->{cpu},
2701             };
2702         } else {
2703             $d->{cpu} = $old->{cpu};
2704         }
2705     }
2706
2707     return $res if !$full;
2708
2709     my $qmpclient = PVE::QMPClient->new();
2710
2711     my $ballooncb = sub {
2712         my ($vmid, $resp) = @_;
2713
2714         my $info = $resp->{'return'};
2715         return if !$info->{max_mem};
2716
2717         my $d = $res->{$vmid};
2718
2719         # use memory assigned to VM
2720         $d->{maxmem} = $info->{max_mem};
2721         $d->{balloon} = $info->{actual};
2722
2723         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2724             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2725             $d->{freemem} = $info->{free_mem};
2726         }
2727
2728         $d->{ballooninfo} = $info;
2729     };
2730
2731     my $blockstatscb = sub {
2732         my ($vmid, $resp) = @_;
2733         my $data = $resp->{'return'} || [];
2734         my $totalrdbytes = 0;
2735         my $totalwrbytes = 0;
2736
2737         for my $blockstat (@$data) {
2738             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2739             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2740
2741             $blockstat->{device} =~ s/drive-//;
2742             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2743         }
2744         $res->{$vmid}->{diskread} = $totalrdbytes;
2745         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2746     };
2747
2748     my $machinecb = sub {
2749         my ($vmid, $resp) = @_;
2750         my $data = $resp->{'return'} || [];
2751
2752         $res->{$vmid}->{'running-machine'} =
2753             PVE::QemuServer::Machine::current_from_query_machines($data);
2754     };
2755
2756     my $versioncb = sub {
2757         my ($vmid, $resp) = @_;
2758         my $data = $resp->{'return'} // {};
2759         my $version = 'unknown';
2760
2761         if (my $v = $data->{qemu}) {
2762             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2763         }
2764
2765         $res->{$vmid}->{'running-qemu'} = $version;
2766     };
2767
2768     my $statuscb = sub {
2769         my ($vmid, $resp) = @_;
2770
2771         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2772         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2773         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2774         # this fails if ballon driver is not loaded, so this must be
2775         # the last commnand (following command are aborted if this fails).
2776         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2777
2778         my $status = 'unknown';
2779         if (!defined($status = $resp->{'return'}->{status})) {
2780             warn "unable to get VM status\n";
2781             return;
2782         }
2783
2784         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2785     };
2786
2787     foreach my $vmid (keys %$list) {
2788         next if $opt_vmid && ($vmid ne $opt_vmid);
2789         next if !$res->{$vmid}->{pid}; # not running
2790         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2791     }
2792
2793     $qmpclient->queue_execute(undef, 2);
2794
2795     foreach my $vmid (keys %$list) {
2796         next if $opt_vmid && ($vmid ne $opt_vmid);
2797         next if !$res->{$vmid}->{pid}; #not running
2798
2799         # we can't use the $qmpclient since it might have already aborted on
2800         # 'query-balloon', but this might also fail for older versions...
2801         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2802         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2803     }
2804
2805     foreach my $vmid (keys %$list) {
2806         next if $opt_vmid && ($vmid ne $opt_vmid);
2807         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2808     }
2809
2810     return $res;
2811 }
2812
2813 sub conf_has_serial {
2814     my ($conf) = @_;
2815
2816     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2817         if ($conf->{"serial$i"}) {
2818             return 1;
2819         }
2820     }
2821
2822     return 0;
2823 }
2824
2825 sub conf_has_audio {
2826     my ($conf, $id) = @_;
2827
2828     $id //= 0;
2829     my $audio = $conf->{"audio$id"};
2830     return if !defined($audio);
2831
2832     my $audioproperties = parse_property_string($audio_fmt, $audio);
2833     my $audiodriver = $audioproperties->{driver} // 'spice';
2834
2835     return {
2836         dev => $audioproperties->{device},
2837         dev_id => "audiodev$id",
2838         backend => $audiodriver,
2839         backend_id => "$audiodriver-backend${id}",
2840     };
2841 }
2842
2843 sub audio_devs {
2844     my ($audio, $audiopciaddr, $machine_version) = @_;
2845
2846     my $devs = [];
2847
2848     my $id = $audio->{dev_id};
2849     my $audiodev = "";
2850     if (min_version($machine_version, 4, 2)) {
2851         $audiodev = ",audiodev=$audio->{backend_id}";
2852     }
2853
2854     if ($audio->{dev} eq 'AC97') {
2855         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2856     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2857         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2858         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2859         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2860     } else {
2861         die "unkown audio device '$audio->{dev}', implement me!";
2862     }
2863
2864     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2865
2866     return $devs;
2867 }
2868
2869 sub vga_conf_has_spice {
2870     my ($vga) = @_;
2871
2872     my $vgaconf = parse_vga($vga);
2873     my $vgatype = $vgaconf->{type};
2874     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2875
2876     return $1 || 1;
2877 }
2878
2879 sub is_native($) {
2880     my ($arch) = @_;
2881     return get_host_arch() eq $arch;
2882 }
2883
2884 sub get_vm_arch {
2885     my ($conf) = @_;
2886     return $conf->{arch} // get_host_arch();
2887 }
2888
2889 my $default_machines = {
2890     x86_64 => 'pc',
2891     aarch64 => 'virt',
2892 };
2893
2894 sub get_installed_machine_version {
2895     my ($kvmversion) = @_;
2896     $kvmversion = kvm_user_version() if !defined($kvmversion);
2897     $kvmversion =~ m/^(\d+\.\d+)/;
2898     return $1;
2899 }
2900
2901 sub windows_get_pinned_machine_version {
2902     my ($machine, $base_version, $kvmversion) = @_;
2903
2904     my $pin_version = $base_version;
2905     if (!defined($base_version) ||
2906         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2907     ) {
2908         $pin_version = get_installed_machine_version($kvmversion);
2909     }
2910     if (!$machine || $machine eq 'pc') {
2911         $machine = "pc-i440fx-$pin_version";
2912     } elsif ($machine eq 'q35') {
2913         $machine = "pc-q35-$pin_version";
2914     } elsif ($machine eq 'virt') {
2915         $machine = "virt-$pin_version";
2916     } else {
2917         warn "unknown machine type '$machine', not touching that!\n";
2918     }
2919
2920     return $machine;
2921 }
2922
2923 sub get_vm_machine {
2924     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2925
2926     my $machine = $forcemachine || $conf->{machine};
2927
2928     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2929         $kvmversion //= kvm_user_version();
2930         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2931         # layout which confuses windows quite a bit and may result in various regressions..
2932         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2933         if (windows_version($conf->{ostype})) {
2934             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2935         }
2936         $arch //= 'x86_64';
2937         $machine ||= $default_machines->{$arch};
2938         if ($add_pve_version) {
2939             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2940             $machine .= "+pve$pvever";
2941         }
2942     }
2943
2944     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2945         # for version-pinned machines that do not include a pve-version (e.g.
2946         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2947         $machine .= '+pve0';
2948     }
2949
2950     return $machine;
2951 }
2952
2953 sub get_ovmf_files($) {
2954     my ($arch) = @_;
2955
2956     my $ovmf = $OVMF->{$arch}
2957         or die "no OVMF images known for architecture '$arch'\n";
2958
2959     return @$ovmf;
2960 }
2961
2962 my $Arch2Qemu = {
2963     aarch64 => '/usr/bin/qemu-system-aarch64',
2964     x86_64 => '/usr/bin/qemu-system-x86_64',
2965 };
2966 sub get_command_for_arch($) {
2967     my ($arch) = @_;
2968     return '/usr/bin/kvm' if is_native($arch);
2969
2970     my $cmd = $Arch2Qemu->{$arch}
2971         or die "don't know how to emulate architecture '$arch'\n";
2972     return $cmd;
2973 }
2974
2975 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2976 # to use in a QEMU command line (-cpu element), first array_intersect the result
2977 # of query_supported_ with query_understood_. This is necessary because:
2978 #
2979 # a) query_understood_ returns flags the host cannot use and
2980 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2981 #    flags, but CPU settings - with most of them being flags. Those settings
2982 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2983 #
2984 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2985 # expensive. If you need the value returned from this, you can get it much
2986 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2987 # $accel being 'kvm' or 'tcg'.
2988 #
2989 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2990 # changes, automatically populating pmxcfs.
2991 #
2992 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2993 # since kvm and tcg machines support different flags
2994 #
2995 sub query_supported_cpu_flags {
2996     my ($arch) = @_;
2997
2998     $arch //= get_host_arch();
2999     my $default_machine = $default_machines->{$arch};
3000
3001     my $flags = {};
3002
3003     # FIXME: Once this is merged, the code below should work for ARM as well:
3004     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3005     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3006         $arch eq "aarch64";
3007
3008     my $kvm_supported = defined(kvm_version());
3009     my $qemu_cmd = get_command_for_arch($arch);
3010     my $fakevmid = -1;
3011     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3012
3013     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3014     my $query_supported_run_qemu = sub {
3015         my ($kvm) = @_;
3016
3017         my $flags = {};
3018         my $cmd = [
3019             $qemu_cmd,
3020             '-machine', $default_machine,
3021             '-display', 'none',
3022             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3023             '-mon', 'chardev=qmp,mode=control',
3024             '-pidfile', $pidfile,
3025             '-S', '-daemonize'
3026         ];
3027
3028         if (!$kvm) {
3029             push @$cmd, '-accel', 'tcg';
3030         }
3031
3032         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3033         die "QEMU flag querying VM exited with code " . $rc if $rc;
3034
3035         eval {
3036             my $cmd_result = mon_cmd(
3037                 $fakevmid,
3038                 'query-cpu-model-expansion',
3039                 type => 'full',
3040                 model => { name => 'host' }
3041             );
3042
3043             my $props = $cmd_result->{model}->{props};
3044             foreach my $prop (keys %$props) {
3045                 next if $props->{$prop} ne '1';
3046                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3047                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3048                 # We only keep those with underscores, to match /proc/cpuinfo
3049                 $prop =~ s/\.|-/_/g;
3050                 $flags->{$prop} = 1;
3051             }
3052         };
3053         my $err = $@;
3054
3055         # force stop with 10 sec timeout and 'nocheck'
3056         # always stop, even if QMP failed
3057         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3058
3059         die $err if $err;
3060
3061         return [ sort keys %$flags ];
3062     };
3063
3064     # We need to query QEMU twice, since KVM and TCG have different supported flags
3065     PVE::QemuConfig->lock_config($fakevmid, sub {
3066         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3067         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3068
3069         if ($kvm_supported) {
3070             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3071             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3072         }
3073     });
3074
3075     return $flags;
3076 }
3077
3078 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3079 my $understood_cpu_flag_dir = "/usr/share/kvm";
3080 sub query_understood_cpu_flags {
3081     my $arch = get_host_arch();
3082     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3083
3084     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3085         if ! -e $filepath;
3086
3087     my $raw = file_get_contents($filepath);
3088     $raw =~ s/^\s+|\s+$//g;
3089     my @flags = split(/\s+/, $raw);
3090
3091     return \@flags;
3092 }
3093
3094 sub config_to_command {
3095     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3096
3097     my $cmd = [];
3098     my $globalFlags = [];
3099     my $machineFlags = [];
3100     my $rtcFlags = [];
3101     my $devices = [];
3102     my $pciaddr = '';
3103     my $bridges = {};
3104     my $ostype = $conf->{ostype};
3105     my $winversion = windows_version($ostype);
3106     my $kvm = $conf->{kvm};
3107     my $nodename = nodename();
3108
3109     my $arch = get_vm_arch($conf);
3110     my $kvm_binary = get_command_for_arch($arch);
3111     my $kvmver = kvm_user_version($kvm_binary);
3112
3113     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3114         $kvmver //= "undefined";
3115         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3116     }
3117
3118     my $add_pve_version = min_version($kvmver, 4, 1);
3119
3120     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3121     my $machine_version = extract_version($machine_type, $kvmver);
3122     $kvm //= 1 if is_native($arch);
3123
3124     $machine_version =~ m/(\d+)\.(\d+)/;
3125     my ($machine_major, $machine_minor) = ($1, $2);
3126
3127     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3128         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3129     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3130         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3131             ." please upgrade node '$nodename'\n"
3132     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3133         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3134         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3135             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3136             ." node '$nodename'\n";
3137     }
3138
3139     # if a specific +pve version is required for a feature, use $version_guard
3140     # instead of min_version to allow machines to be run with the minimum
3141     # required version
3142     my $required_pve_version = 0;
3143     my $version_guard = sub {
3144         my ($major, $minor, $pve) = @_;
3145         return 0 if !min_version($machine_version, $major, $minor, $pve);
3146         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3147         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3148         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3149         return 1;
3150     };
3151
3152     if ($kvm && !defined kvm_version()) {
3153         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3154             ." or enable in BIOS.\n";
3155     }
3156
3157     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3158     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3159     my $use_old_bios_files = undef;
3160     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3161
3162     my $cpuunits = defined($conf->{cpuunits}) ?
3163             $conf->{cpuunits} : $defaults->{cpuunits};
3164
3165     push @$cmd, $kvm_binary;
3166
3167     push @$cmd, '-id', $vmid;
3168
3169     my $vmname = $conf->{name} || "vm$vmid";
3170
3171     push @$cmd, '-name', $vmname;
3172
3173     push @$cmd, '-no-shutdown';
3174
3175     my $use_virtio = 0;
3176
3177     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3178     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3179     push @$cmd, '-mon', "chardev=qmp,mode=control";
3180
3181     if (min_version($machine_version, 2, 12)) {
3182         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3183         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3184     }
3185
3186     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3187
3188     push @$cmd, '-daemonize';
3189
3190     if ($conf->{smbios1}) {
3191         my $smbios_conf = parse_smbios1($conf->{smbios1});
3192         if ($smbios_conf->{base64}) {
3193             # Do not pass base64 flag to qemu
3194             delete $smbios_conf->{base64};
3195             my $smbios_string = "";
3196             foreach my $key (keys %$smbios_conf) {
3197                 my $value;
3198                 if ($key eq "uuid") {
3199                     $value = $smbios_conf->{uuid}
3200                 } else {
3201                     $value = decode_base64($smbios_conf->{$key});
3202                 }
3203                 # qemu accepts any binary data, only commas need escaping by double comma
3204                 $value =~ s/,/,,/g;
3205                 $smbios_string .= "," . $key . "=" . $value if $value;
3206             }
3207             push @$cmd, '-smbios', "type=1" . $smbios_string;
3208         } else {
3209             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3210         }
3211     }
3212
3213     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3214         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3215         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3216
3217         my ($path, $format);
3218         if (my $efidisk = $conf->{efidisk0}) {
3219             my $d = parse_drive('efidisk0', $efidisk);
3220             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3221             $format = $d->{format};
3222             if ($storeid) {
3223                 $path = PVE::Storage::path($storecfg, $d->{file});
3224                 if (!defined($format)) {
3225                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3226                     $format = qemu_img_format($scfg, $volname);
3227                 }
3228             } else {
3229                 $path = $d->{file};
3230                 die "efidisk format must be specified\n"
3231                     if !defined($format);
3232             }
3233         } else {
3234             warn "no efidisk configured! Using temporary efivars disk.\n";
3235             $path = "/tmp/$vmid-ovmf.fd";
3236             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3237             $format = 'raw';
3238         }
3239
3240         my $size_str = "";
3241
3242         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3243             $size_str = ",size=" . (-s $ovmf_vars);
3244         }
3245
3246         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3247         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3248     }
3249
3250     # load q35 config
3251     if ($q35) {
3252         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3253         if (min_version($machine_version, 4, 0)) {
3254             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3255         } else {
3256             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3257         }
3258     }
3259
3260     if ($conf->{vmgenid}) {
3261         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3262     }
3263
3264     # add usb controllers
3265     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3266         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3267     push @$devices, @usbcontrollers if @usbcontrollers;
3268     my $vga = parse_vga($conf->{vga});
3269
3270     my $qxlnum = vga_conf_has_spice($conf->{vga});
3271     $vga->{type} = 'qxl' if $qxlnum;
3272
3273     if (!$vga->{type}) {
3274         if ($arch eq 'aarch64') {
3275             $vga->{type} = 'virtio';
3276         } elsif (min_version($machine_version, 2, 9)) {
3277             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3278         } else {
3279             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3280         }
3281     }
3282
3283     # enable absolute mouse coordinates (needed by vnc)
3284     my $tablet;
3285     if (defined($conf->{tablet})) {
3286         $tablet = $conf->{tablet};
3287     } else {
3288         $tablet = $defaults->{tablet};
3289         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3290         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3291     }
3292
3293     if ($tablet) {
3294         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3295         my $kbd = print_keyboarddevice_full($conf, $arch);
3296         push @$devices, '-device', $kbd if defined($kbd);
3297     }
3298
3299     my $bootorder = device_bootorder($conf);
3300
3301     # host pci device passthrough
3302     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3303         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3304
3305     # usb devices
3306     my $usb_dev_features = {};
3307     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3308
3309     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3310         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3311     push @$devices, @usbdevices if @usbdevices;
3312
3313     # serial devices
3314     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3315         if (my $path = $conf->{"serial$i"}) {
3316             if ($path eq 'socket') {
3317                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3318                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3319                 # On aarch64, serial0 is the UART device. Qemu only allows
3320                 # connecting UART devices via the '-serial' command line, as
3321                 # the device has a fixed slot on the hardware...
3322                 if ($arch eq 'aarch64' && $i == 0) {
3323                     push @$devices, '-serial', "chardev:serial$i";
3324                 } else {
3325                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3326                 }
3327             } else {
3328                 die "no such serial device\n" if ! -c $path;
3329                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3330                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3331             }
3332         }
3333     }
3334
3335     # parallel devices
3336     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3337         if (my $path = $conf->{"parallel$i"}) {
3338             die "no such parallel device\n" if ! -c $path;
3339             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3340             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3341             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3342         }
3343     }
3344
3345     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3346         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3347         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3348         push @$devices, @$audio_devs;
3349     }
3350
3351     my $sockets = 1;
3352     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3353     $sockets = $conf->{sockets} if  $conf->{sockets};
3354
3355     my $cores = $conf->{cores} || 1;
3356
3357     my $maxcpus = $sockets * $cores;
3358
3359     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3360
3361     my $allowed_vcpus = $cpuinfo->{cpus};
3362
3363     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3364         if ($allowed_vcpus < $maxcpus);
3365
3366     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3367
3368         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3369         for (my $i = 2; $i <= $vcpus; $i++)  {
3370             my $cpustr = print_cpu_device($conf,$i);
3371             push @$cmd, '-device', $cpustr;
3372         }
3373
3374     } else {
3375
3376         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3377     }
3378     push @$cmd, '-nodefaults';
3379
3380     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3381
3382     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3383
3384     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3385
3386     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3387         push @$devices, '-device', print_vga_device(
3388             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3389         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3390         push @$cmd,  '-vnc', "unix:$socket,password";
3391     } else {
3392         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3393         push @$cmd, '-nographic';
3394     }
3395
3396     # time drift fix
3397     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3398     my $useLocaltime = $conf->{localtime};
3399
3400     if ($winversion >= 5) { # windows
3401         $useLocaltime = 1 if !defined($conf->{localtime});
3402
3403         # use time drift fix when acpi is enabled
3404         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3405             $tdf = 1 if !defined($conf->{tdf});
3406         }
3407     }
3408
3409     if ($winversion >= 6) {
3410         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3411         push @$cmd, '-no-hpet';
3412     }
3413
3414     push @$rtcFlags, 'driftfix=slew' if $tdf;
3415
3416     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3417         push @$rtcFlags, "base=$conf->{startdate}";
3418     } elsif ($useLocaltime) {
3419         push @$rtcFlags, 'base=localtime';
3420     }
3421
3422     if ($forcecpu) {
3423         push @$cmd, '-cpu', $forcecpu;
3424     } else {
3425         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3426     }
3427
3428     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3429
3430     push @$cmd, '-S' if $conf->{freeze};
3431
3432     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3433
3434     my $guest_agent = parse_guest_agent($conf);
3435
3436     if ($guest_agent->{enabled}) {
3437         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3438         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3439
3440         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3441             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3442             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3443             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3444         } elsif ($guest_agent->{type} eq 'isa') {
3445             push @$devices, '-device', "isa-serial,chardev=qga0";
3446         }
3447     }
3448
3449     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3450     if ($rng && $version_guard->(4, 1, 2)) {
3451         check_rng_source($rng->{source});
3452
3453         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3454         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3455         my $limiter_str = "";
3456         if ($max_bytes) {
3457             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3458         }
3459
3460         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3461         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3462         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3463     }
3464
3465     my $spice_port;
3466
3467     if ($qxlnum) {
3468         if ($qxlnum > 1) {
3469             if ($winversion){
3470                 for (my $i = 1; $i < $qxlnum; $i++){
3471                     push @$devices, '-device', print_vga_device(
3472                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3473                 }
3474             } else {
3475                 # assume other OS works like Linux
3476                 my ($ram, $vram) = ("134217728", "67108864");
3477                 if ($vga->{memory}) {
3478                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3479                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3480                 }
3481                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3482                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3483             }
3484         }
3485
3486         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3487
3488         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3489         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3490         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3491
3492         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3493         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3494         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3495
3496         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3497         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3498
3499         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3500         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3501         if ($spice_enhancement->{foldersharing}) {
3502             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3503             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3504         }
3505
3506         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3507         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3508             if $spice_enhancement->{videostreaming};
3509
3510         push @$devices, '-spice', "$spice_opts";
3511     }
3512
3513     # enable balloon by default, unless explicitly disabled
3514     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3515         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3516         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3517     }
3518
3519     if ($conf->{watchdog}) {
3520         my $wdopts = parse_watchdog($conf->{watchdog});
3521         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3522         my $watchdog = $wdopts->{model} || 'i6300esb';
3523         push @$devices, '-device', "$watchdog$pciaddr";
3524         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3525     }
3526
3527     my $vollist = [];
3528     my $scsicontroller = {};
3529     my $ahcicontroller = {};
3530     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3531
3532     # Add iscsi initiator name if available
3533     if (my $initiator = get_initiator_name()) {
3534         push @$devices, '-iscsi', "initiator-name=$initiator";
3535     }
3536
3537     PVE::QemuConfig->foreach_volume($conf, sub {
3538         my ($ds, $drive) = @_;
3539
3540         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3541             push @$vollist, $drive->{file};
3542         }
3543
3544         # ignore efidisk here, already added in bios/fw handling code above
3545         return if $drive->{interface} eq 'efidisk';
3546
3547         $use_virtio = 1 if $ds =~ m/^virtio/;
3548
3549         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3550
3551         if ($drive->{interface} eq 'virtio'){
3552            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3553         }
3554
3555         if ($drive->{interface} eq 'scsi') {
3556
3557             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3558
3559             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3560                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3561
3562             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3563             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3564
3565             my $iothread = '';
3566             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3567                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3568                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3569             } elsif ($drive->{iothread}) {
3570                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3571             }
3572
3573             my $queues = '';
3574             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3575                 $queues = ",num_queues=$drive->{queues}";
3576             }
3577
3578             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3579                 if !$scsicontroller->{$controller};
3580             $scsicontroller->{$controller}=1;
3581         }
3582
3583         if ($drive->{interface} eq 'sata') {
3584             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3585             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3586             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3587                 if !$ahcicontroller->{$controller};
3588             $ahcicontroller->{$controller}=1;
3589         }
3590
3591         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3592         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3593
3594         push @$devices, '-drive',$drive_cmd;
3595         push @$devices, '-device', print_drivedevice_full(
3596             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3597     });
3598
3599     for (my $i = 0; $i < $MAX_NETS; $i++) {
3600         my $netname = "net$i";
3601
3602         next if !$conf->{$netname};
3603         my $d = parse_net($conf->{$netname});
3604         next if !$d;
3605
3606         $use_virtio = 1 if $d->{model} eq 'virtio';
3607
3608         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3609
3610         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3611         push @$devices, '-netdev', $netdevfull;
3612
3613         my $netdevicefull = print_netdevice_full(
3614             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3615
3616         push @$devices, '-device', $netdevicefull;
3617     }
3618
3619     if ($conf->{ivshmem}) {
3620         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3621
3622         my $bus;
3623         if ($q35) {
3624             $bus = print_pcie_addr("ivshmem");
3625         } else {
3626             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3627         }
3628
3629         my $ivshmem_name = $ivshmem->{name} // $vmid;
3630         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3631
3632         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3633         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3634             .",size=$ivshmem->{size}M";
3635     }
3636
3637     # pci.4 is nested in pci.1
3638     $bridges->{1} = 1 if $bridges->{4};
3639
3640     if (!$q35) {
3641         # add pci bridges
3642         if (min_version($machine_version, 2, 3)) {
3643            $bridges->{1} = 1;
3644            $bridges->{2} = 1;
3645         }
3646
3647         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3648
3649     }
3650
3651     for my $k (sort {$b cmp $a} keys %$bridges) {
3652         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3653
3654         my $k_name = $k;
3655         if ($k == 2 && $legacy_igd) {
3656             $k_name = "$k-igd";
3657         }
3658         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3659
3660         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3661         if ($q35) {
3662             # add after -readconfig pve-q35.cfg
3663             splice @$devices, 2, 0, '-device', $devstr;
3664         } else {
3665             unshift @$devices, '-device', $devstr if $k > 0;
3666         }
3667     }
3668
3669     if (!$kvm) {
3670         push @$machineFlags, 'accel=tcg';
3671     }
3672
3673     my $machine_type_min = $machine_type;
3674     if ($add_pve_version) {
3675         $machine_type_min =~ s/\+pve\d+$//;
3676         $machine_type_min .= "+pve$required_pve_version";
3677     }
3678     push @$machineFlags, "type=${machine_type_min}";
3679
3680     push @$cmd, @$devices;
3681     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3682     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3683     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3684
3685     if (my $vmstate = $conf->{vmstate}) {
3686         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3687         push @$vollist, $vmstate;
3688         push @$cmd, '-loadstate', $statepath;
3689         print "activating and using '$vmstate' as vmstate\n";
3690     }
3691
3692     # add custom args
3693     if ($conf->{args}) {
3694         my $aa = PVE::Tools::split_args($conf->{args});
3695         push @$cmd, @$aa;
3696     }
3697
3698     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3699 }
3700
3701 sub check_rng_source {
3702     my ($source) = @_;
3703
3704     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3705     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3706         if ! -e $source;
3707
3708     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3709     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3710         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3711         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3712         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3713             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3714             ." to the host.\n";
3715     }
3716 }
3717
3718 sub spice_port {
3719     my ($vmid) = @_;
3720
3721     my $res = mon_cmd($vmid, 'query-spice');
3722
3723     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3724 }
3725
3726 sub vm_devices_list {
3727     my ($vmid) = @_;
3728
3729     my $res = mon_cmd($vmid, 'query-pci');
3730     my $devices_to_check = [];
3731     my $devices = {};
3732     foreach my $pcibus (@$res) {
3733         push @$devices_to_check, @{$pcibus->{devices}},
3734     }
3735
3736     while (@$devices_to_check) {
3737         my $to_check = [];
3738         for my $d (@$devices_to_check) {
3739             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3740             next if !$d->{'pci_bridge'};
3741
3742             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3743             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3744         }
3745         $devices_to_check = $to_check;
3746     }
3747
3748     my $resblock = mon_cmd($vmid, 'query-block');
3749     foreach my $block (@$resblock) {
3750         if($block->{device} =~ m/^drive-(\S+)/){
3751                 $devices->{$1} = 1;
3752         }
3753     }
3754
3755     my $resmice = mon_cmd($vmid, 'query-mice');
3756     foreach my $mice (@$resmice) {
3757         if ($mice->{name} eq 'QEMU HID Tablet') {
3758             $devices->{tablet} = 1;
3759             last;
3760         }
3761     }
3762
3763     # for usb devices there is no query-usb
3764     # but we can iterate over the entries in
3765     # qom-list path=/machine/peripheral
3766     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3767     foreach my $per (@$resperipheral) {
3768         if ($per->{name} =~ m/^usb\d+$/) {
3769             $devices->{$per->{name}} = 1;
3770         }
3771     }
3772
3773     return $devices;
3774 }
3775
3776 sub vm_deviceplug {
3777     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3778
3779     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3780
3781     my $devices_list = vm_devices_list($vmid);
3782     return 1 if defined($devices_list->{$deviceid});
3783
3784     # add PCI bridge if we need it for the device
3785     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3786
3787     if ($deviceid eq 'tablet') {
3788
3789         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3790
3791     } elsif ($deviceid eq 'keyboard') {
3792
3793         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3794
3795     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3796
3797         die "usb hotplug currently not reliable\n";
3798         # since we can't reliably hot unplug all added usb devices and usb
3799         # passthrough breaks live migration we disable usb hotplugging for now
3800         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3801
3802     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3803
3804         qemu_iothread_add($vmid, $deviceid, $device);
3805
3806         qemu_driveadd($storecfg, $vmid, $device);
3807         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3808
3809         qemu_deviceadd($vmid, $devicefull);
3810         eval { qemu_deviceaddverify($vmid, $deviceid); };
3811         if (my $err = $@) {
3812             eval { qemu_drivedel($vmid, $deviceid); };
3813             warn $@ if $@;
3814             die $err;
3815         }
3816
3817     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3818
3819
3820         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3821         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3822         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3823
3824         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3825
3826         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3827             qemu_iothread_add($vmid, $deviceid, $device);
3828             $devicefull .= ",iothread=iothread-$deviceid";
3829         }
3830
3831         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3832             $devicefull .= ",num_queues=$device->{queues}";
3833         }
3834
3835         qemu_deviceadd($vmid, $devicefull);
3836         qemu_deviceaddverify($vmid, $deviceid);
3837
3838     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3839
3840         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3841         qemu_driveadd($storecfg, $vmid, $device);
3842
3843         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3844         eval { qemu_deviceadd($vmid, $devicefull); };
3845         if (my $err = $@) {
3846             eval { qemu_drivedel($vmid, $deviceid); };
3847             warn $@ if $@;
3848             die $err;
3849         }
3850
3851     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3852
3853         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3854
3855         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3856         my $use_old_bios_files = undef;
3857         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3858
3859         my $netdevicefull = print_netdevice_full(
3860             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3861         qemu_deviceadd($vmid, $netdevicefull);
3862         eval {
3863             qemu_deviceaddverify($vmid, $deviceid);
3864             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3865         };
3866         if (my $err = $@) {
3867             eval { qemu_netdevdel($vmid, $deviceid); };
3868             warn $@ if $@;
3869             die $err;
3870         }
3871
3872     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3873
3874         my $bridgeid = $2;
3875         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3876         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3877
3878         qemu_deviceadd($vmid, $devicefull);
3879         qemu_deviceaddverify($vmid, $deviceid);
3880
3881     } else {
3882         die "can't hotplug device '$deviceid'\n";
3883     }
3884
3885     return 1;
3886 }
3887
3888 # fixme: this should raise exceptions on error!
3889 sub vm_deviceunplug {
3890     my ($vmid, $conf, $deviceid) = @_;
3891
3892     my $devices_list = vm_devices_list($vmid);
3893     return 1 if !defined($devices_list->{$deviceid});
3894
3895     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3896     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3897
3898     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3899
3900         qemu_devicedel($vmid, $deviceid);
3901
3902     } elsif ($deviceid =~ m/^usb\d+$/) {
3903
3904         die "usb hotplug currently not reliable\n";
3905         # when unplugging usb devices this way, there may be remaining usb
3906         # controllers/hubs so we disable it for now
3907         #qemu_devicedel($vmid, $deviceid);
3908         #qemu_devicedelverify($vmid, $deviceid);
3909
3910     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3911
3912         qemu_devicedel($vmid, $deviceid);
3913         qemu_devicedelverify($vmid, $deviceid);
3914         qemu_drivedel($vmid, $deviceid);
3915         qemu_iothread_del($conf, $vmid, $deviceid);
3916
3917     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3918
3919         qemu_devicedel($vmid, $deviceid);
3920         qemu_devicedelverify($vmid, $deviceid);
3921         qemu_iothread_del($conf, $vmid, $deviceid);
3922
3923     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3924
3925         qemu_devicedel($vmid, $deviceid);
3926         qemu_drivedel($vmid, $deviceid);
3927         qemu_deletescsihw($conf, $vmid, $deviceid);
3928
3929     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3930
3931         qemu_devicedel($vmid, $deviceid);
3932         qemu_devicedelverify($vmid, $deviceid);
3933         qemu_netdevdel($vmid, $deviceid);
3934
3935     } else {
3936         die "can't unplug device '$deviceid'\n";
3937     }
3938
3939     return 1;
3940 }
3941
3942 sub qemu_deviceadd {
3943     my ($vmid, $devicefull) = @_;
3944
3945     $devicefull = "driver=".$devicefull;
3946     my %options =  split(/[=,]/, $devicefull);
3947
3948     mon_cmd($vmid, "device_add" , %options);
3949 }
3950
3951 sub qemu_devicedel {
3952     my ($vmid, $deviceid) = @_;
3953
3954     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3955 }
3956
3957 sub qemu_iothread_add {
3958     my($vmid, $deviceid, $device) = @_;
3959
3960     if ($device->{iothread}) {
3961         my $iothreads = vm_iothreads_list($vmid);
3962         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3963     }
3964 }
3965
3966 sub qemu_iothread_del {
3967     my($conf, $vmid, $deviceid) = @_;
3968
3969     my $confid = $deviceid;
3970     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3971         $confid = 'scsi' . $1;
3972     }
3973     my $device = parse_drive($confid, $conf->{$confid});
3974     if ($device->{iothread}) {
3975         my $iothreads = vm_iothreads_list($vmid);
3976         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3977     }
3978 }
3979
3980 sub qemu_objectadd {
3981     my($vmid, $objectid, $qomtype) = @_;
3982
3983     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3984
3985     return 1;
3986 }
3987
3988 sub qemu_objectdel {
3989     my($vmid, $objectid) = @_;
3990
3991     mon_cmd($vmid, "object-del", id => $objectid);
3992
3993     return 1;
3994 }
3995
3996 sub qemu_driveadd {
3997     my ($storecfg, $vmid, $device) = @_;
3998
3999     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4000     $drive =~ s/\\/\\\\/g;
4001     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4002
4003     # If the command succeeds qemu prints: "OK"
4004     return 1 if $ret =~ m/OK/s;
4005
4006     die "adding drive failed: $ret\n";
4007 }
4008
4009 sub qemu_drivedel {
4010     my($vmid, $deviceid) = @_;
4011
4012     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4013     $ret =~ s/^\s+//;
4014
4015     return 1 if $ret eq "";
4016
4017     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4018     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4019
4020     die "deleting drive $deviceid failed : $ret\n";
4021 }
4022
4023 sub qemu_deviceaddverify {
4024     my ($vmid, $deviceid) = @_;
4025
4026     for (my $i = 0; $i <= 5; $i++) {
4027          my $devices_list = vm_devices_list($vmid);
4028          return 1 if defined($devices_list->{$deviceid});
4029          sleep 1;
4030     }
4031
4032     die "error on hotplug device '$deviceid'\n";
4033 }
4034
4035
4036 sub qemu_devicedelverify {
4037     my ($vmid, $deviceid) = @_;
4038
4039     # need to verify that the device is correctly removed as device_del
4040     # is async and empty return is not reliable
4041
4042     for (my $i = 0; $i <= 5; $i++) {
4043          my $devices_list = vm_devices_list($vmid);
4044          return 1 if !defined($devices_list->{$deviceid});
4045          sleep 1;
4046     }
4047
4048     die "error on hot-unplugging device '$deviceid'\n";
4049 }
4050
4051 sub qemu_findorcreatescsihw {
4052     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4053
4054     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4055
4056     my $scsihwid="$controller_prefix$controller";
4057     my $devices_list = vm_devices_list($vmid);
4058
4059     if(!defined($devices_list->{$scsihwid})) {
4060         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4061     }
4062
4063     return 1;
4064 }
4065
4066 sub qemu_deletescsihw {
4067     my ($conf, $vmid, $opt) = @_;
4068
4069     my $device = parse_drive($opt, $conf->{$opt});
4070
4071     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4072         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4073         return 1;
4074     }
4075
4076     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4077
4078     my $devices_list = vm_devices_list($vmid);
4079     foreach my $opt (keys %{$devices_list}) {
4080         if (is_valid_drivename($opt)) {
4081             my $drive = parse_drive($opt, $conf->{$opt});
4082             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4083                 return 1;
4084             }
4085         }
4086     }
4087
4088     my $scsihwid="scsihw$controller";
4089
4090     vm_deviceunplug($vmid, $conf, $scsihwid);
4091
4092     return 1;
4093 }
4094
4095 sub qemu_add_pci_bridge {
4096     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4097
4098     my $bridges = {};
4099
4100     my $bridgeid;
4101
4102     print_pci_addr($device, $bridges, $arch, $machine_type);
4103
4104     while (my ($k, $v) = each %$bridges) {
4105         $bridgeid = $k;
4106     }
4107     return 1 if !defined($bridgeid) || $bridgeid < 1;
4108
4109     my $bridge = "pci.$bridgeid";
4110     my $devices_list = vm_devices_list($vmid);
4111
4112     if (!defined($devices_list->{$bridge})) {
4113         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4114     }
4115
4116     return 1;
4117 }
4118
4119 sub qemu_set_link_status {
4120     my ($vmid, $device, $up) = @_;
4121
4122     mon_cmd($vmid, "set_link", name => $device,
4123                up => $up ? JSON::true : JSON::false);
4124 }
4125
4126 sub qemu_netdevadd {
4127     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4128
4129     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4130     my %options =  split(/[=,]/, $netdev);
4131
4132     if (defined(my $vhost = $options{vhost})) {
4133         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4134     }
4135
4136     if (defined(my $queues = $options{queues})) {
4137         $options{queues} = $queues + 0;
4138     }
4139
4140     mon_cmd($vmid, "netdev_add",  %options);
4141     return 1;
4142 }
4143
4144 sub qemu_netdevdel {
4145     my ($vmid, $deviceid) = @_;
4146
4147     mon_cmd($vmid, "netdev_del", id => $deviceid);
4148 }
4149
4150 sub qemu_usb_hotplug {
4151     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4152
4153     return if !$device;
4154
4155     # remove the old one first
4156     vm_deviceunplug($vmid, $conf, $deviceid);
4157
4158     # check if xhci controller is necessary and available
4159     if ($device->{usb3}) {
4160
4161         my $devicelist = vm_devices_list($vmid);
4162
4163         if (!$devicelist->{xhci}) {
4164             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4165             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4166         }
4167     }
4168     my $d = parse_usb_device($device->{host});
4169     $d->{usb3} = $device->{usb3};
4170
4171     # add the new one
4172     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4173 }
4174
4175 sub qemu_cpu_hotplug {
4176     my ($vmid, $conf, $vcpus) = @_;
4177
4178     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4179
4180     my $sockets = 1;
4181     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4182     $sockets = $conf->{sockets} if  $conf->{sockets};
4183     my $cores = $conf->{cores} || 1;
4184     my $maxcpus = $sockets * $cores;
4185
4186     $vcpus = $maxcpus if !$vcpus;
4187
4188     die "you can't add more vcpus than maxcpus\n"
4189         if $vcpus > $maxcpus;
4190
4191     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4192
4193     if ($vcpus < $currentvcpus) {
4194
4195         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4196
4197             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4198                 qemu_devicedel($vmid, "cpu$i");
4199                 my $retry = 0;
4200                 my $currentrunningvcpus = undef;
4201                 while (1) {
4202                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4203                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4204                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4205                     $retry++;
4206                     sleep 1;
4207                 }
4208                 #update conf after each succesfull cpu unplug
4209                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4210                 PVE::QemuConfig->write_config($vmid, $conf);
4211             }
4212         } else {
4213             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4214         }
4215
4216         return;
4217     }
4218
4219     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4220     die "vcpus in running vm does not match its configuration\n"
4221         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4222
4223     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4224
4225         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4226             my $cpustr = print_cpu_device($conf, $i);
4227             qemu_deviceadd($vmid, $cpustr);
4228
4229             my $retry = 0;
4230             my $currentrunningvcpus = undef;
4231             while (1) {
4232                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4233                 last if scalar(@{$currentrunningvcpus}) == $i;
4234                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4235                 sleep 1;
4236                 $retry++;
4237             }
4238             #update conf after each succesfull cpu hotplug
4239             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4240             PVE::QemuConfig->write_config($vmid, $conf);
4241         }
4242     } else {
4243
4244         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4245             mon_cmd($vmid, "cpu-add", id => int($i));
4246         }
4247     }
4248 }
4249
4250 sub qemu_block_set_io_throttle {
4251     my ($vmid, $deviceid,
4252         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4253         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4254         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4255         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4256
4257     return if !check_running($vmid) ;
4258
4259     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4260         bps => int($bps),
4261         bps_rd => int($bps_rd),
4262         bps_wr => int($bps_wr),
4263         iops => int($iops),
4264         iops_rd => int($iops_rd),
4265         iops_wr => int($iops_wr),
4266         bps_max => int($bps_max),
4267         bps_rd_max => int($bps_rd_max),
4268         bps_wr_max => int($bps_wr_max),
4269         iops_max => int($iops_max),
4270         iops_rd_max => int($iops_rd_max),
4271         iops_wr_max => int($iops_wr_max),
4272         bps_max_length => int($bps_max_length),
4273         bps_rd_max_length => int($bps_rd_max_length),
4274         bps_wr_max_length => int($bps_wr_max_length),
4275         iops_max_length => int($iops_max_length),
4276         iops_rd_max_length => int($iops_rd_max_length),
4277         iops_wr_max_length => int($iops_wr_max_length),
4278     );
4279
4280 }
4281
4282 sub qemu_block_resize {
4283     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4284
4285     my $running = check_running($vmid);
4286
4287     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4288
4289     return if !$running;
4290
4291     my $padding = (1024 - $size % 1024) % 1024;
4292     $size = $size + $padding;
4293
4294     mon_cmd(
4295         $vmid,
4296         "block_resize",
4297         device => $deviceid,
4298         size => int($size),
4299         timeout => 60,
4300     );
4301 }
4302
4303 sub qemu_volume_snapshot {
4304     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4305
4306     my $running = check_running($vmid);
4307
4308     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4309         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4310     } else {
4311         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4312     }
4313 }
4314
4315 sub qemu_volume_snapshot_delete {
4316     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4317
4318     my $running = check_running($vmid);
4319
4320     if($running) {
4321
4322         $running = undef;
4323         my $conf = PVE::QemuConfig->load_config($vmid);
4324         PVE::QemuConfig->foreach_volume($conf, sub {
4325             my ($ds, $drive) = @_;
4326             $running = 1 if $drive->{file} eq $volid;
4327         });
4328     }
4329
4330     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4331         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4332     } else {
4333         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4334     }
4335 }
4336
4337 sub set_migration_caps {
4338     my ($vmid, $savevm) = @_;
4339
4340     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4341
4342     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4343     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4344
4345     my $cap_ref = [];
4346
4347     my $enabled_cap = {
4348         "auto-converge" => 1,
4349         "xbzrle" => 1,
4350         "x-rdma-pin-all" => 0,
4351         "zero-blocks" => 0,
4352         "compress" => 0,
4353         "dirty-bitmaps" => $dirty_bitmaps,
4354     };
4355
4356     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4357
4358     for my $supported_capability (@$supported_capabilities) {
4359         push @$cap_ref, {
4360             capability => $supported_capability->{capability},
4361             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4362         };
4363     }
4364
4365     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4366 }
4367
4368 sub foreach_volid {
4369     my ($conf, $func, @param) = @_;
4370
4371     my $volhash = {};
4372
4373     my $test_volid = sub {
4374         my ($key, $drive, $snapname) = @_;
4375
4376         my $volid = $drive->{file};
4377         return if !$volid;
4378
4379         $volhash->{$volid}->{cdrom} //= 1;
4380         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4381
4382         my $replicate = $drive->{replicate} // 1;
4383         $volhash->{$volid}->{replicate} //= 0;
4384         $volhash->{$volid}->{replicate} = 1 if $replicate;
4385
4386         $volhash->{$volid}->{shared} //= 0;
4387         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4388
4389         $volhash->{$volid}->{referenced_in_config} //= 0;
4390         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4391
4392         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4393             if defined($snapname);
4394
4395         my $size = $drive->{size};
4396         $volhash->{$volid}->{size} //= $size if $size;
4397
4398         $volhash->{$volid}->{is_vmstate} //= 0;
4399         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4400
4401         $volhash->{$volid}->{is_unused} //= 0;
4402         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4403     };
4404
4405     my $include_opts = {
4406         extra_keys => ['vmstate'],
4407         include_unused => 1,
4408     };
4409
4410     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4411     foreach my $snapname (keys %{$conf->{snapshots}}) {
4412         my $snap = $conf->{snapshots}->{$snapname};
4413         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4414     }
4415
4416     foreach my $volid (keys %$volhash) {
4417         &$func($volid, $volhash->{$volid}, @param);
4418     }
4419 }
4420
4421 my $fast_plug_option = {
4422     'lock' => 1,
4423     'name' => 1,
4424     'onboot' => 1,
4425     'shares' => 1,
4426     'startup' => 1,
4427     'description' => 1,
4428     'protection' => 1,
4429     'vmstatestorage' => 1,
4430     'hookscript' => 1,
4431     'tags' => 1,
4432 };
4433
4434 # hotplug changes in [PENDING]
4435 # $selection hash can be used to only apply specified options, for
4436 # example: { cores => 1 } (only apply changed 'cores')
4437 # $errors ref is used to return error messages
4438 sub vmconfig_hotplug_pending {
4439     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4440
4441     my $defaults = load_defaults();
4442     my $arch = get_vm_arch($conf);
4443     my $machine_type = get_vm_machine($conf, undef, $arch);
4444
4445     # commit values which do not have any impact on running VM first
4446     # Note: those option cannot raise errors, we we do not care about
4447     # $selection and always apply them.
4448
4449     my $add_error = sub {
4450         my ($opt, $msg) = @_;
4451         $errors->{$opt} = "hotplug problem - $msg";
4452     };
4453
4454     my $changes = 0;
4455     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4456         if ($fast_plug_option->{$opt}) {
4457             $conf->{$opt} = $conf->{pending}->{$opt};
4458             delete $conf->{pending}->{$opt};
4459             $changes = 1;
4460         }
4461     }
4462
4463     if ($changes) {
4464         PVE::QemuConfig->write_config($vmid, $conf);
4465     }
4466
4467     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4468
4469     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4470     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4471     foreach my $opt (sort keys %$pending_delete_hash) {
4472         next if $selection && !$selection->{$opt};
4473         my $force = $pending_delete_hash->{$opt}->{force};
4474         eval {
4475             if ($opt eq 'hotplug') {
4476                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4477             } elsif ($opt eq 'tablet') {
4478                 die "skip\n" if !$hotplug_features->{usb};
4479                 if ($defaults->{tablet}) {
4480                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4481                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4482                         if $arch eq 'aarch64';
4483                 } else {
4484                     vm_deviceunplug($vmid, $conf, 'tablet');
4485                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4486                 }
4487             } elsif ($opt =~ m/^usb\d+/) {
4488                 die "skip\n";
4489                 # since we cannot reliably hot unplug usb devices we are disabling it
4490                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4491                 #vm_deviceunplug($vmid, $conf, $opt);
4492             } elsif ($opt eq 'vcpus') {
4493                 die "skip\n" if !$hotplug_features->{cpu};
4494                 qemu_cpu_hotplug($vmid, $conf, undef);
4495             } elsif ($opt eq 'balloon') {
4496                 # enable balloon device is not hotpluggable
4497                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4498                 # here we reset the ballooning value to memory
4499                 my $balloon = $conf->{memory} || $defaults->{memory};
4500                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4501             } elsif ($fast_plug_option->{$opt}) {
4502                 # do nothing
4503             } elsif ($opt =~ m/^net(\d+)$/) {
4504                 die "skip\n" if !$hotplug_features->{network};
4505                 vm_deviceunplug($vmid, $conf, $opt);
4506             } elsif (is_valid_drivename($opt)) {
4507                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4508                 vm_deviceunplug($vmid, $conf, $opt);
4509                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4510             } elsif ($opt =~ m/^memory$/) {
4511                 die "skip\n" if !$hotplug_features->{memory};
4512                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4513             } elsif ($opt eq 'cpuunits') {
4514                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4515             } elsif ($opt eq 'cpulimit') {
4516                 $cgroup->change_cpu_quota(-1, 100000);
4517             } else {
4518                 die "skip\n";
4519             }
4520         };
4521         if (my $err = $@) {
4522             &$add_error($opt, $err) if $err ne "skip\n";
4523         } else {
4524             delete $conf->{$opt};
4525             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4526         }
4527     }
4528
4529     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4530     $apply_pending_cloudinit = sub {
4531         return if $apply_pending_cloudinit_done; # once is enough
4532         $apply_pending_cloudinit_done = 1; # once is enough
4533
4534         my ($key, $value) = @_;
4535
4536         my @cloudinit_opts = keys %$confdesc_cloudinit;
4537         foreach my $opt (keys %{$conf->{pending}}) {
4538             next if !grep { $_ eq $opt } @cloudinit_opts;
4539             $conf->{$opt} = delete $conf->{pending}->{$opt};
4540         }
4541
4542         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4543         foreach my $opt (sort keys %$pending_delete_hash) {
4544             next if !grep { $_ eq $opt } @cloudinit_opts;
4545             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4546             delete $conf->{$opt};
4547         }
4548
4549         my $new_conf = { %$conf };
4550         $new_conf->{$key} = $value;
4551         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4552     };
4553
4554     foreach my $opt (keys %{$conf->{pending}}) {
4555         next if $selection && !$selection->{$opt};
4556         my $value = $conf->{pending}->{$opt};
4557         eval {
4558             if ($opt eq 'hotplug') {
4559                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4560             } elsif ($opt eq 'tablet') {
4561                 die "skip\n" if !$hotplug_features->{usb};
4562                 if ($value == 1) {
4563                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4564                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4565                         if $arch eq 'aarch64';
4566                 } elsif ($value == 0) {
4567                     vm_deviceunplug($vmid, $conf, 'tablet');
4568                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4569                 }
4570             } elsif ($opt =~ m/^usb\d+$/) {
4571                 die "skip\n";
4572                 # since we cannot reliably hot unplug usb devices we disable it for now
4573                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4574                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4575                 #die "skip\n" if !$d;
4576                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4577             } elsif ($opt eq 'vcpus') {
4578                 die "skip\n" if !$hotplug_features->{cpu};
4579                 qemu_cpu_hotplug($vmid, $conf, $value);
4580             } elsif ($opt eq 'balloon') {
4581                 # enable/disable balloning device is not hotpluggable
4582                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4583                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4584                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4585
4586                 # allow manual ballooning if shares is set to zero
4587                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4588                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4589                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4590                 }
4591             } elsif ($opt =~ m/^net(\d+)$/) {
4592                 # some changes can be done without hotplug
4593                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4594                                     $vmid, $opt, $value, $arch, $machine_type);
4595             } elsif (is_valid_drivename($opt)) {
4596                 die "skip\n" if $opt eq 'efidisk0';
4597                 # some changes can be done without hotplug
4598                 my $drive = parse_drive($opt, $value);
4599                 if (drive_is_cloudinit($drive)) {
4600                     &$apply_pending_cloudinit($opt, $value);
4601                 }
4602                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4603                                      $vmid, $opt, $value, $arch, $machine_type);
4604             } elsif ($opt =~ m/^memory$/) { #dimms
4605                 die "skip\n" if !$hotplug_features->{memory};
4606                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4607             } elsif ($opt eq 'cpuunits') {
4608                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4609             } elsif ($opt eq 'cpulimit') {
4610                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4611                 $cgroup->change_cpu_quota($cpulimit, 100000);
4612             } else {
4613                 die "skip\n";  # skip non-hot-pluggable options
4614             }
4615         };
4616         if (my $err = $@) {
4617             &$add_error($opt, $err) if $err ne "skip\n";
4618         } else {
4619             $conf->{$opt} = $value;
4620             delete $conf->{pending}->{$opt};
4621         }
4622     }
4623
4624     PVE::QemuConfig->write_config($vmid, $conf);
4625 }
4626
4627 sub try_deallocate_drive {
4628     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4629
4630     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4631         my $volid = $drive->{file};
4632         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4633             my $sid = PVE::Storage::parse_volume_id($volid);
4634             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4635
4636             # check if the disk is really unused
4637             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4638                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4639             PVE::Storage::vdisk_free($storecfg, $volid);
4640             return 1;
4641         } else {
4642             # If vm is not owner of this disk remove from config
4643             return 1;
4644         }
4645     }
4646
4647     return;
4648 }
4649
4650 sub vmconfig_delete_or_detach_drive {
4651     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4652
4653     my $drive = parse_drive($opt, $conf->{$opt});
4654
4655     my $rpcenv = PVE::RPCEnvironment::get();
4656     my $authuser = $rpcenv->get_user();
4657
4658     if ($force) {
4659         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4660         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4661     } else {
4662         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4663     }
4664 }
4665
4666
4667
4668 sub vmconfig_apply_pending {
4669     my ($vmid, $conf, $storecfg, $errors) = @_;
4670
4671     my $add_apply_error = sub {
4672         my ($opt, $msg) = @_;
4673         my $err_msg = "unable to apply pending change $opt : $msg";
4674         $errors->{$opt} = $err_msg;
4675         warn $err_msg;
4676     };
4677
4678     # cold plug
4679
4680     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4681     foreach my $opt (sort keys %$pending_delete_hash) {
4682         my $force = $pending_delete_hash->{$opt}->{force};
4683         eval {
4684             if ($opt =~ m/^unused/) {
4685                 die "internal error";
4686             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4687                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4688             }
4689         };
4690         if (my $err = $@) {
4691             $add_apply_error->($opt, $err);
4692         } else {
4693             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4694             delete $conf->{$opt};
4695         }
4696     }
4697
4698     PVE::QemuConfig->cleanup_pending($conf);
4699
4700     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4701         next if $opt eq 'delete'; # just to be sure
4702         eval {
4703             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4704                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4705             }
4706         };
4707         if (my $err = $@) {
4708             $add_apply_error->($opt, $err);
4709         } else {
4710             $conf->{$opt} = delete $conf->{pending}->{$opt};
4711         }
4712     }
4713
4714     # write all changes at once to avoid unnecessary i/o
4715     PVE::QemuConfig->write_config($vmid, $conf);
4716 }
4717
4718 sub vmconfig_update_net {
4719     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4720
4721     my $newnet = parse_net($value);
4722
4723     if ($conf->{$opt}) {
4724         my $oldnet = parse_net($conf->{$opt});
4725
4726         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4727             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4728             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4729             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4730
4731             # for non online change, we try to hot-unplug
4732             die "skip\n" if !$hotplug;
4733             vm_deviceunplug($vmid, $conf, $opt);
4734         } else {
4735
4736             die "internal error" if $opt !~ m/net(\d+)/;
4737             my $iface = "tap${vmid}i$1";
4738
4739             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4740                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4741                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4742                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4743                 PVE::Network::tap_unplug($iface);
4744
4745                 if ($have_sdn) {
4746                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4747                 } else {
4748                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4749                 }
4750             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4751                 # Rate can be applied on its own but any change above needs to
4752                 # include the rate in tap_plug since OVS resets everything.
4753                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4754             }
4755
4756             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4757                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4758             }
4759
4760             return 1;
4761         }
4762     }
4763
4764     if ($hotplug) {
4765         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4766     } else {
4767         die "skip\n";
4768     }
4769 }
4770
4771 sub vmconfig_update_disk {
4772     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4773
4774     my $drive = parse_drive($opt, $value);
4775
4776     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4777         my $media = $drive->{media} || 'disk';
4778         my $oldmedia = $old_drive->{media} || 'disk';
4779         die "unable to change media type\n" if $media ne $oldmedia;
4780
4781         if (!drive_is_cdrom($old_drive)) {
4782
4783             if ($drive->{file} ne $old_drive->{file}) {
4784
4785                 die "skip\n" if !$hotplug;
4786
4787                 # unplug and register as unused
4788                 vm_deviceunplug($vmid, $conf, $opt);
4789                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4790
4791             } else {
4792                 # update existing disk
4793
4794                 # skip non hotpluggable value
4795                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4796                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4797                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4798                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4799                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4800                     die "skip\n";
4801                 }
4802
4803                 # apply throttle
4804                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4805                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4806                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4807                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4808                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4809                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4810                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4811                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4812                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4813                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4814                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4815                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4816                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4817                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4818                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4819                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4820                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4821                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4822
4823                     qemu_block_set_io_throttle(
4824                         $vmid,"drive-$opt",
4825                         ($drive->{mbps} || 0)*1024*1024,
4826                         ($drive->{mbps_rd} || 0)*1024*1024,
4827                         ($drive->{mbps_wr} || 0)*1024*1024,
4828                          $drive->{iops} || 0,
4829                          $drive->{iops_rd} || 0,
4830                          $drive->{iops_wr} || 0,
4831                         ($drive->{mbps_max} || 0)*1024*1024,
4832                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4833                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4834                          $drive->{iops_max} || 0,
4835                          $drive->{iops_rd_max} || 0,
4836                          $drive->{iops_wr_max} || 0,
4837                          $drive->{bps_max_length} || 1,
4838                          $drive->{bps_rd_max_length} || 1,
4839                          $drive->{bps_wr_max_length} || 1,
4840                          $drive->{iops_max_length} || 1,
4841                          $drive->{iops_rd_max_length} || 1,
4842                          $drive->{iops_wr_max_length} || 1,
4843                     );
4844
4845                 }
4846
4847                 return 1;
4848             }
4849
4850         } else { # cdrom
4851
4852             if ($drive->{file} eq 'none') {
4853                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4854                 if (drive_is_cloudinit($old_drive)) {
4855                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4856                 }
4857             } else {
4858                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4859
4860                 # force eject if locked
4861                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4862
4863                 if ($path) {
4864                     mon_cmd($vmid, "blockdev-change-medium",
4865                         id => "$opt", filename => "$path");
4866                 }
4867             }
4868
4869             return 1;
4870         }
4871     }
4872
4873     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4874     # hotplug new disks
4875     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4876     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4877 }
4878
4879 # called in locked context by incoming migration
4880 sub vm_migrate_get_nbd_disks {
4881     my ($storecfg, $conf, $replicated_volumes) = @_;
4882
4883     my $local_volumes = {};
4884     PVE::QemuConfig->foreach_volume($conf, sub {
4885         my ($ds, $drive) = @_;
4886
4887         return if drive_is_cdrom($drive);
4888
4889         my $volid = $drive->{file};
4890
4891         return if !$volid;
4892
4893         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4894
4895         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4896         return if $scfg->{shared};
4897
4898         # replicated disks re-use existing state via bitmap
4899         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4900         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4901     });
4902     return $local_volumes;
4903 }
4904
4905 # called in locked context by incoming migration
4906 sub vm_migrate_alloc_nbd_disks {
4907     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4908
4909     my $format = undef;
4910
4911     my $nbd = {};
4912     foreach my $opt (sort keys %$source_volumes) {
4913         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4914
4915         if ($use_existing) {
4916             $nbd->{$opt}->{drivestr} = print_drive($drive);
4917             $nbd->{$opt}->{volid} = $volid;
4918             $nbd->{$opt}->{replicated} = 1;
4919             next;
4920         }
4921
4922         # If a remote storage is specified and the format of the original
4923         # volume is not available there, fall back to the default format.
4924         # Otherwise use the same format as the original.
4925         if (!$storagemap->{identity}) {
4926             $storeid = map_storage($storagemap, $storeid);
4927             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4928             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4929             my $fileFormat = qemu_img_format($scfg, $volname);
4930             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4931         } else {
4932             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4933             $format = qemu_img_format($scfg, $volname);
4934         }
4935
4936         my $size = $drive->{size} / 1024;
4937         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4938         my $newdrive = $drive;
4939         $newdrive->{format} = $format;
4940         $newdrive->{file} = $newvolid;
4941         my $drivestr = print_drive($newdrive);
4942         $nbd->{$opt}->{drivestr} = $drivestr;
4943         $nbd->{$opt}->{volid} = $newvolid;
4944     }
4945
4946     return $nbd;
4947 }
4948
4949 # see vm_start_nolock for parameters, additionally:
4950 # migrate_opts:
4951 #   storagemap = parsed storage map for allocating NBD disks
4952 sub vm_start {
4953     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4954
4955     return PVE::QemuConfig->lock_config($vmid, sub {
4956         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4957
4958         die "you can't start a vm if it's a template\n"
4959             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4960
4961         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4962         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4963
4964         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4965
4966         if ($has_backup_lock && $running) {
4967             # a backup is currently running, attempt to start the guest in the
4968             # existing QEMU instance
4969             return vm_resume($vmid);
4970         }
4971
4972         PVE::QemuConfig->check_lock($conf)
4973             if !($params->{skiplock} || $has_suspended_lock);
4974
4975         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4976
4977         die "VM $vmid already running\n" if $running;
4978
4979         if (my $storagemap = $migrate_opts->{storagemap}) {
4980             my $replicated = $migrate_opts->{replicated_volumes};
4981             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4982             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4983
4984             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4985                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4986             }
4987         }
4988
4989         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4990     });
4991 }
4992
4993
4994 # params:
4995 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4996 #   skiplock => 0/1, skip checking for config lock
4997 #   skiptemplate => 0/1, skip checking whether VM is template
4998 #   forcemachine => to force Qemu machine (rollback/migration)
4999 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5000 #   timeout => in seconds
5001 #   paused => start VM in paused state (backup)
5002 #   resume => resume from hibernation
5003 # migrate_opts:
5004 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5005 #   migratedfrom => source node
5006 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5007 #   network => CIDR of migration network
5008 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5009 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5010 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5011 sub vm_start_nolock {
5012     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5013
5014     my $statefile = $params->{statefile};
5015     my $resume = $params->{resume};
5016
5017     my $migratedfrom = $migrate_opts->{migratedfrom};
5018     my $migration_type = $migrate_opts->{type};
5019
5020     my $res = {};
5021
5022     # clean up leftover reboot request files
5023     eval { clear_reboot_request($vmid); };
5024     warn $@ if $@;
5025
5026     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5027         vmconfig_apply_pending($vmid, $conf, $storecfg);
5028         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5029     }
5030
5031     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5032
5033     my $defaults = load_defaults();
5034
5035     # set environment variable useful inside network script
5036     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5037
5038     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5039
5040     my $forcemachine = $params->{forcemachine};
5041     my $forcecpu = $params->{forcecpu};
5042     if ($resume) {
5043         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5044         $forcemachine = $conf->{runningmachine};
5045         $forcecpu = $conf->{runningcpu};
5046         print "Resuming suspended VM\n";
5047     }
5048
5049     my ($cmd, $vollist, $spice_port) =
5050         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5051
5052     my $migration_ip;
5053     my $get_migration_ip = sub {
5054         my ($nodename) = @_;
5055
5056         return $migration_ip if defined($migration_ip);
5057
5058         my $cidr = $migrate_opts->{network};
5059
5060         if (!defined($cidr)) {
5061             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5062             $cidr = $dc_conf->{migration}->{network};
5063         }
5064
5065         if (defined($cidr)) {
5066             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5067
5068             die "could not get IP: no address configured on local " .
5069                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5070
5071             die "could not get IP: multiple addresses configured on local " .
5072                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5073
5074             $migration_ip = @$ips[0];
5075         }
5076
5077         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5078             if !defined($migration_ip);
5079
5080         return $migration_ip;
5081     };
5082
5083     my $migrate_uri;
5084     if ($statefile) {
5085         if ($statefile eq 'tcp') {
5086             my $localip = "localhost";
5087             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5088             my $nodename = nodename();
5089
5090             if (!defined($migration_type)) {
5091                 if (defined($datacenterconf->{migration}->{type})) {
5092                     $migration_type = $datacenterconf->{migration}->{type};
5093                 } else {
5094                     $migration_type = 'secure';
5095                 }
5096             }
5097
5098             if ($migration_type eq 'insecure') {
5099                 $localip = $get_migration_ip->($nodename);
5100                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5101             }
5102
5103             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5104             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5105             $migrate_uri = "tcp:${localip}:${migrate_port}";
5106             push @$cmd, '-incoming', $migrate_uri;
5107             push @$cmd, '-S';
5108
5109         } elsif ($statefile eq 'unix') {
5110             # should be default for secure migrations as a ssh TCP forward
5111             # tunnel is not deterministic reliable ready and fails regurarly
5112             # to set up in time, so use UNIX socket forwards
5113             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5114             unlink $socket_addr;
5115
5116             $migrate_uri = "unix:$socket_addr";
5117
5118             push @$cmd, '-incoming', $migrate_uri;
5119             push @$cmd, '-S';
5120
5121         } elsif (-e $statefile) {
5122             push @$cmd, '-loadstate', $statefile;
5123         } else {
5124             my $statepath = PVE::Storage::path($storecfg, $statefile);
5125             push @$vollist, $statefile;
5126             push @$cmd, '-loadstate', $statepath;
5127         }
5128     } elsif ($params->{paused}) {
5129         push @$cmd, '-S';
5130     }
5131
5132     # host pci devices
5133     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5134       my $d = parse_hostpci($conf->{"hostpci$i"});
5135       next if !$d;
5136       my $pcidevices = $d->{pciid};
5137       foreach my $pcidevice (@$pcidevices) {
5138             my $pciid = $pcidevice->{id};
5139
5140             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5141             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5142             die "no pci device info for device '$pciid'\n" if !$info;
5143
5144             if ($d->{mdev}) {
5145                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5146                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5147             } else {
5148                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5149                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5150                 die "can't reset PCI device '$pciid'\n"
5151                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5152             }
5153       }
5154     }
5155
5156     PVE::Storage::activate_volumes($storecfg, $vollist);
5157
5158     eval {
5159         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5160             outfunc => sub {}, errfunc => sub {});
5161     };
5162     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5163     # timeout should be more than enough here...
5164     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5165
5166     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5167                                               : $defaults->{cpuunits};
5168
5169     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5170     my %run_params = (
5171         timeout => $statefile ? undef : $start_timeout,
5172         umask => 0077,
5173         noerr => 1,
5174     );
5175
5176     # when migrating, prefix QEMU output so other side can pick up any
5177     # errors that might occur and show the user
5178     if ($migratedfrom) {
5179         $run_params{quiet} = 1;
5180         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5181     }
5182
5183     my %properties = (
5184         Slice => 'qemu.slice',
5185         KillMode => 'none'
5186     );
5187
5188     if (PVE::CGroup::cgroup_mode() == 2) {
5189         $properties{CPUWeight} = $cpuunits;
5190     } else {
5191         $properties{CPUShares} = $cpuunits;
5192     }
5193
5194     if (my $cpulimit = $conf->{cpulimit}) {
5195         $properties{CPUQuota} = int($cpulimit * 100);
5196     }
5197     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5198
5199     my $run_qemu = sub {
5200         PVE::Tools::run_fork sub {
5201             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5202
5203             my $exitcode = run_command($cmd, %run_params);
5204             die "QEMU exited with code $exitcode\n" if $exitcode;
5205         };
5206     };
5207
5208     if ($conf->{hugepages}) {
5209
5210         my $code = sub {
5211             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5212             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5213
5214             PVE::QemuServer::Memory::hugepages_mount();
5215             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5216
5217             eval { $run_qemu->() };
5218             if (my $err = $@) {
5219                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5220                     if !$conf->{keephugepages};
5221                 die $err;
5222             }
5223
5224             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5225                 if !$conf->{keephugepages};
5226         };
5227         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5228
5229     } else {
5230         eval { $run_qemu->() };
5231     }
5232
5233     if (my $err = $@) {
5234         # deactivate volumes if start fails
5235         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5236         die "start failed: $err";
5237     }
5238
5239     print "migration listens on $migrate_uri\n" if $migrate_uri;
5240     $res->{migrate_uri} = $migrate_uri;
5241
5242     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5243         eval { mon_cmd($vmid, "cont"); };
5244         warn $@ if $@;
5245     }
5246
5247     #start nbd server for storage migration
5248     if (my $nbd = $migrate_opts->{nbd}) {
5249         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5250
5251         my $migrate_storage_uri;
5252         # nbd_protocol_version > 0 for unix socket support
5253         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5254             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5255             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5256             $migrate_storage_uri = "nbd:unix:$socket_path";
5257         } else {
5258             my $nodename = nodename();
5259             my $localip = $get_migration_ip->($nodename);
5260             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5261             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5262
5263             mon_cmd($vmid, "nbd-server-start", addr => {
5264                 type => 'inet',
5265                 data => {
5266                     host => "${localip}",
5267                     port => "${storage_migrate_port}",
5268                 },
5269             });
5270             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5271             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5272         }
5273
5274         $res->{migrate_storage_uri} = $migrate_storage_uri;
5275
5276         foreach my $opt (sort keys %$nbd) {
5277             my $drivestr = $nbd->{$opt}->{drivestr};
5278             my $volid = $nbd->{$opt}->{volid};
5279             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5280             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5281             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5282             print "re-using replicated volume: $opt - $volid\n"
5283                 if $nbd->{$opt}->{replicated};
5284
5285             $res->{drives}->{$opt} = $nbd->{$opt};
5286             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5287         }
5288     }
5289
5290     if ($migratedfrom) {
5291         eval {
5292             set_migration_caps($vmid);
5293         };
5294         warn $@ if $@;
5295
5296         if ($spice_port) {
5297             print "spice listens on port $spice_port\n";
5298             $res->{spice_port} = $spice_port;
5299             if ($migrate_opts->{spice_ticket}) {
5300                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5301                     $migrate_opts->{spice_ticket});
5302                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5303             }
5304         }
5305
5306     } else {
5307         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5308             if !$statefile && $conf->{balloon};
5309
5310         foreach my $opt (keys %$conf) {
5311             next if $opt !~  m/^net\d+$/;
5312             my $nicconf = parse_net($conf->{$opt});
5313             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5314         }
5315     }
5316
5317     mon_cmd($vmid, 'qom-set',
5318                 path => "machine/peripheral/balloon0",
5319                 property => "guest-stats-polling-interval",
5320                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5321
5322     if ($resume) {
5323         print "Resumed VM, removing state\n";
5324         if (my $vmstate = $conf->{vmstate}) {
5325             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5326             PVE::Storage::vdisk_free($storecfg, $vmstate);
5327         }
5328         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5329         PVE::QemuConfig->write_config($vmid, $conf);
5330     }
5331
5332     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5333
5334     return $res;
5335 }
5336
5337 sub vm_commandline {
5338     my ($storecfg, $vmid, $snapname) = @_;
5339
5340     my $conf = PVE::QemuConfig->load_config($vmid);
5341     my $forcemachine;
5342     my $forcecpu;
5343
5344     if ($snapname) {
5345         my $snapshot = $conf->{snapshots}->{$snapname};
5346         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5347
5348         # check for machine or CPU overrides in snapshot
5349         $forcemachine = $snapshot->{runningmachine};
5350         $forcecpu = $snapshot->{runningcpu};
5351
5352         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5353
5354         $conf = $snapshot;
5355     }
5356
5357     my $defaults = load_defaults();
5358
5359     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5360         $forcemachine, $forcecpu);
5361
5362     return PVE::Tools::cmd2string($cmd);
5363 }
5364
5365 sub vm_reset {
5366     my ($vmid, $skiplock) = @_;
5367
5368     PVE::QemuConfig->lock_config($vmid, sub {
5369
5370         my $conf = PVE::QemuConfig->load_config($vmid);
5371
5372         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5373
5374         mon_cmd($vmid, "system_reset");
5375     });
5376 }
5377
5378 sub get_vm_volumes {
5379     my ($conf) = @_;
5380
5381     my $vollist = [];
5382     foreach_volid($conf, sub {
5383         my ($volid, $attr) = @_;
5384
5385         return if $volid =~ m|^/|;
5386
5387         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5388         return if !$sid;
5389
5390         push @$vollist, $volid;
5391     });
5392
5393     return $vollist;
5394 }
5395
5396 sub vm_stop_cleanup {
5397     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5398
5399     eval {
5400
5401         if (!$keepActive) {
5402             my $vollist = get_vm_volumes($conf);
5403             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5404         }
5405
5406         foreach my $ext (qw(mon qmp pid vnc qga)) {
5407             unlink "/var/run/qemu-server/${vmid}.$ext";
5408         }
5409
5410         if ($conf->{ivshmem}) {
5411             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5412             # just delete it for now, VMs which have this already open do not
5413             # are affected, but new VMs will get a separated one. If this
5414             # becomes an issue we either add some sort of ref-counting or just
5415             # add a "don't delete on stop" flag to the ivshmem format.
5416             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5417         }
5418
5419         foreach my $key (keys %$conf) {
5420             next if $key !~ m/^hostpci(\d+)$/;
5421             my $hostpciindex = $1;
5422             my $d = parse_hostpci($conf->{$key});
5423             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5424
5425             foreach my $pci (@{$d->{pciid}}) {
5426                 my $pciid = $pci->{id};
5427                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5428             }
5429         }
5430
5431         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5432     };
5433     warn $@ if $@; # avoid errors - just warn
5434 }
5435
5436 # call only in locked context
5437 sub _do_vm_stop {
5438     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5439
5440     my $pid = check_running($vmid, $nocheck);
5441     return if !$pid;
5442
5443     my $conf;
5444     if (!$nocheck) {
5445         $conf = PVE::QemuConfig->load_config($vmid);
5446         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5447         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5448             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5449             $timeout = $opts->{down} if $opts->{down};
5450         }
5451         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5452     }
5453
5454     eval {
5455         if ($shutdown) {
5456             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5457                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5458             } else {
5459                 mon_cmd($vmid, "system_powerdown");
5460             }
5461         } else {
5462             mon_cmd($vmid, "quit");
5463         }
5464     };
5465     my $err = $@;
5466
5467     if (!$err) {
5468         $timeout = 60 if !defined($timeout);
5469
5470         my $count = 0;
5471         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5472             $count++;
5473             sleep 1;
5474         }
5475
5476         if ($count >= $timeout) {
5477             if ($force) {
5478                 warn "VM still running - terminating now with SIGTERM\n";
5479                 kill 15, $pid;
5480             } else {
5481                 die "VM quit/powerdown failed - got timeout\n";
5482             }
5483         } else {
5484             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5485             return;
5486         }
5487     } else {
5488        if (!check_running($vmid, $nocheck)) {
5489             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5490             return;
5491        }
5492        if ($force) {
5493             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5494             kill 15, $pid;
5495         } else {
5496             die "VM quit/powerdown failed\n";
5497         }
5498     }
5499
5500     # wait again
5501     $timeout = 10;
5502
5503     my $count = 0;
5504     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5505         $count++;
5506         sleep 1;
5507     }
5508
5509     if ($count >= $timeout) {
5510         warn "VM still running - terminating now with SIGKILL\n";
5511         kill 9, $pid;
5512         sleep 1;
5513     }
5514
5515     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5516 }
5517
5518 # Note: use $nocheck to skip tests if VM configuration file exists.
5519 # We need that when migration VMs to other nodes (files already moved)
5520 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5521 sub vm_stop {
5522     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5523
5524     $force = 1 if !defined($force) && !$shutdown;
5525
5526     if ($migratedfrom){
5527         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5528         kill 15, $pid if $pid;
5529         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5530         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5531         return;
5532     }
5533
5534     PVE::QemuConfig->lock_config($vmid, sub {
5535         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5536    });
5537 }
5538
5539 sub vm_reboot {
5540     my ($vmid, $timeout) = @_;
5541
5542     PVE::QemuConfig->lock_config($vmid, sub {
5543         eval {
5544
5545             # only reboot if running, as qmeventd starts it again on a stop event
5546             return if !check_running($vmid);
5547
5548             create_reboot_request($vmid);
5549
5550             my $storecfg = PVE::Storage::config();
5551             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5552
5553         };
5554         if (my $err = $@) {
5555             # avoid that the next normal shutdown will be confused for a reboot
5556             clear_reboot_request($vmid);
5557             die $err;
5558         }
5559    });
5560 }
5561
5562 # note: if using the statestorage parameter, the caller has to check privileges
5563 sub vm_suspend {
5564     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5565
5566     my $conf;
5567     my $path;
5568     my $storecfg;
5569     my $vmstate;
5570
5571     PVE::QemuConfig->lock_config($vmid, sub {
5572
5573         $conf = PVE::QemuConfig->load_config($vmid);
5574
5575         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5576         PVE::QemuConfig->check_lock($conf)
5577             if !($skiplock || $is_backing_up);
5578
5579         die "cannot suspend to disk during backup\n"
5580             if $is_backing_up && $includestate;
5581
5582         if ($includestate) {
5583             $conf->{lock} = 'suspending';
5584             my $date = strftime("%Y-%m-%d", localtime(time()));
5585             $storecfg = PVE::Storage::config();
5586             if (!$statestorage) {
5587                 $statestorage = find_vmstate_storage($conf, $storecfg);
5588                 # check permissions for the storage
5589                 my $rpcenv = PVE::RPCEnvironment::get();
5590                 if ($rpcenv->{type} ne 'cli') {
5591                     my $authuser = $rpcenv->get_user();
5592                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5593                 }
5594             }
5595
5596
5597             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5598                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5599             $path = PVE::Storage::path($storecfg, $vmstate);
5600             PVE::QemuConfig->write_config($vmid, $conf);
5601         } else {
5602             mon_cmd($vmid, "stop");
5603         }
5604     });
5605
5606     if ($includestate) {
5607         # save vm state
5608         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5609
5610         eval {
5611             set_migration_caps($vmid, 1);
5612             mon_cmd($vmid, "savevm-start", statefile => $path);
5613             for(;;) {
5614                 my $state = mon_cmd($vmid, "query-savevm");
5615                 if (!$state->{status}) {
5616                     die "savevm not active\n";
5617                 } elsif ($state->{status} eq 'active') {
5618                     sleep(1);
5619                     next;
5620                 } elsif ($state->{status} eq 'completed') {
5621                     print "State saved, quitting\n";
5622                     last;
5623                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5624                     die "query-savevm failed with error '$state->{error}'\n"
5625                 } else {
5626                     die "query-savevm returned status '$state->{status}'\n";
5627                 }
5628             }
5629         };
5630         my $err = $@;
5631
5632         PVE::QemuConfig->lock_config($vmid, sub {
5633             $conf = PVE::QemuConfig->load_config($vmid);
5634             if ($err) {
5635                 # cleanup, but leave suspending lock, to indicate something went wrong
5636                 eval {
5637                     mon_cmd($vmid, "savevm-end");
5638                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5639                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5640                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5641                     PVE::QemuConfig->write_config($vmid, $conf);
5642                 };
5643                 warn $@ if $@;
5644                 die $err;
5645             }
5646
5647             die "lock changed unexpectedly\n"
5648                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5649
5650             mon_cmd($vmid, "quit");
5651             $conf->{lock} = 'suspended';
5652             PVE::QemuConfig->write_config($vmid, $conf);
5653         });
5654     }
5655 }
5656
5657 sub vm_resume {
5658     my ($vmid, $skiplock, $nocheck) = @_;
5659
5660     PVE::QemuConfig->lock_config($vmid, sub {
5661         my $res = mon_cmd($vmid, 'query-status');
5662         my $resume_cmd = 'cont';
5663         my $reset = 0;
5664
5665         if ($res->{status}) {
5666             return if $res->{status} eq 'running'; # job done, go home
5667             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5668             $reset = 1 if $res->{status} eq 'shutdown';
5669         }
5670
5671         if (!$nocheck) {
5672
5673             my $conf = PVE::QemuConfig->load_config($vmid);
5674
5675             PVE::QemuConfig->check_lock($conf)
5676                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5677         }
5678
5679         if ($reset) {
5680             # required if a VM shuts down during a backup and we get a resume
5681             # request before the backup finishes for example
5682             mon_cmd($vmid, "system_reset");
5683         }
5684         mon_cmd($vmid, $resume_cmd);
5685     });
5686 }
5687
5688 sub vm_sendkey {
5689     my ($vmid, $skiplock, $key) = @_;
5690
5691     PVE::QemuConfig->lock_config($vmid, sub {
5692
5693         my $conf = PVE::QemuConfig->load_config($vmid);
5694
5695         # there is no qmp command, so we use the human monitor command
5696         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5697         die $res if $res ne '';
5698     });
5699 }
5700
5701 # vzdump restore implementaion
5702
5703 sub tar_archive_read_firstfile {
5704     my $archive = shift;
5705
5706     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5707
5708     # try to detect archive type first
5709     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5710         die "unable to open file '$archive'\n";
5711     my $firstfile = <$fh>;
5712     kill 15, $pid;
5713     close $fh;
5714
5715     die "ERROR: archive contaions no data\n" if !$firstfile;
5716     chomp $firstfile;
5717
5718     return $firstfile;
5719 }
5720
5721 sub tar_restore_cleanup {
5722     my ($storecfg, $statfile) = @_;
5723
5724     print STDERR "starting cleanup\n";
5725
5726     if (my $fd = IO::File->new($statfile, "r")) {
5727         while (defined(my $line = <$fd>)) {
5728             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5729                 my $volid = $2;
5730                 eval {
5731                     if ($volid =~ m|^/|) {
5732                         unlink $volid || die 'unlink failed\n';
5733                     } else {
5734                         PVE::Storage::vdisk_free($storecfg, $volid);
5735                     }
5736                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5737                 };
5738                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5739             } else {
5740                 print STDERR "unable to parse line in statfile - $line";
5741             }
5742         }
5743         $fd->close();
5744     }
5745 }
5746
5747 sub restore_file_archive {
5748     my ($archive, $vmid, $user, $opts) = @_;
5749
5750     return restore_vma_archive($archive, $vmid, $user, $opts)
5751         if $archive eq '-';
5752
5753     my $info = PVE::Storage::archive_info($archive);
5754     my $format = $opts->{format} // $info->{format};
5755     my $comp = $info->{compression};
5756
5757     # try to detect archive format
5758     if ($format eq 'tar') {
5759         return restore_tar_archive($archive, $vmid, $user, $opts);
5760     } else {
5761         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5762     }
5763 }
5764
5765 # hepler to remove disks that will not be used after restore
5766 my $restore_cleanup_oldconf = sub {
5767     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5768
5769     PVE::QemuConfig->foreach_volume($oldconf, sub {
5770         my ($ds, $drive) = @_;
5771
5772         return if drive_is_cdrom($drive, 1);
5773
5774         my $volid = $drive->{file};
5775         return if !$volid || $volid =~ m|^/|;
5776
5777         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5778         return if !$path || !$owner || ($owner != $vmid);
5779
5780         # Note: only delete disk we want to restore
5781         # other volumes will become unused
5782         if ($virtdev_hash->{$ds}) {
5783             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5784             if (my $err = $@) {
5785                 warn $err;
5786             }
5787         }
5788     });
5789
5790     # delete vmstate files, after the restore we have no snapshots anymore
5791     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5792         my $snap = $oldconf->{snapshots}->{$snapname};
5793         if ($snap->{vmstate}) {
5794             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5795             if (my $err = $@) {
5796                 warn $err;
5797             }
5798         }
5799     }
5800 };
5801
5802 # Helper to parse vzdump backup device hints
5803 #
5804 # $rpcenv: Environment, used to ckeck storage permissions
5805 # $user: User ID, to check storage permissions
5806 # $storecfg: Storage configuration
5807 # $fh: the file handle for reading the configuration
5808 # $devinfo: should contain device sizes for all backu-up'ed devices
5809 # $options: backup options (pool, default storage)
5810 #
5811 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5812 my $parse_backup_hints = sub {
5813     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5814
5815     my $virtdev_hash = {};
5816
5817     while (defined(my $line = <$fh>)) {
5818         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5819             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5820             die "archive does not contain data for drive '$virtdev'\n"
5821                 if !$devinfo->{$devname};
5822
5823             if (defined($options->{storage})) {
5824                 $storeid = $options->{storage} || 'local';
5825             } elsif (!$storeid) {
5826                 $storeid = 'local';
5827             }
5828             $format = 'raw' if !$format;
5829             $devinfo->{$devname}->{devname} = $devname;
5830             $devinfo->{$devname}->{virtdev} = $virtdev;
5831             $devinfo->{$devname}->{format} = $format;
5832             $devinfo->{$devname}->{storeid} = $storeid;
5833
5834             # check permission on storage
5835             my $pool = $options->{pool}; # todo: do we need that?
5836             if ($user ne 'root@pam') {
5837                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5838             }
5839
5840             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5841         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5842             my $virtdev = $1;
5843             my $drive = parse_drive($virtdev, $2);
5844             if (drive_is_cloudinit($drive)) {
5845                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5846                 $storeid = $options->{storage} if defined ($options->{storage});
5847                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5848                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5849
5850                 $virtdev_hash->{$virtdev} = {
5851                     format => $format,
5852                     storeid => $storeid,
5853                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5854                     is_cloudinit => 1,
5855                 };
5856             }
5857         }
5858     }
5859
5860     return $virtdev_hash;
5861 };
5862
5863 # Helper to allocate and activate all volumes required for a restore
5864 #
5865 # $storecfg: Storage configuration
5866 # $virtdev_hash: as returned by parse_backup_hints()
5867 #
5868 # Returns: { $virtdev => $volid }
5869 my $restore_allocate_devices = sub {
5870     my ($storecfg, $virtdev_hash, $vmid) = @_;
5871
5872     my $map = {};
5873     foreach my $virtdev (sort keys %$virtdev_hash) {
5874         my $d = $virtdev_hash->{$virtdev};
5875         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5876         my $storeid = $d->{storeid};
5877         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5878
5879         # test if requested format is supported
5880         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5881         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5882         $d->{format} = $defFormat if !$supported;
5883
5884         my $name;
5885         if ($d->{is_cloudinit}) {
5886             $name = "vm-$vmid-cloudinit";
5887             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5888             if ($scfg->{path}) {
5889                 $name .= ".$d->{format}";
5890             }
5891         }
5892
5893         my $volid = PVE::Storage::vdisk_alloc(
5894             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5895
5896         print STDERR "new volume ID is '$volid'\n";
5897         $d->{volid} = $volid;
5898
5899         PVE::Storage::activate_volumes($storecfg, [$volid]);
5900
5901         $map->{$virtdev} = $volid;
5902     }
5903
5904     return $map;
5905 };
5906
5907 my $restore_update_config_line = sub {
5908     my ($cookie, $vmid, $map, $line, $unique) = @_;
5909
5910     return '' if $line =~ m/^\#qmdump\#/;
5911     return '' if $line =~ m/^\#vzdump\#/;
5912     return '' if $line =~ m/^lock:/;
5913     return '' if $line =~ m/^unused\d+:/;
5914     return '' if $line =~ m/^parent:/;
5915
5916     my $res = '';
5917
5918     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5919     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5920         # try to convert old 1.X settings
5921         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5922         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5923             my ($model, $macaddr) = split(/\=/, $devconfig);
5924             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5925             my $net = {
5926                 model => $model,
5927                 bridge => "vmbr$ind",
5928                 macaddr => $macaddr,
5929             };
5930             my $netstr = print_net($net);
5931
5932             $res .= "net$cookie->{netcount}: $netstr\n";
5933             $cookie->{netcount}++;
5934         }
5935     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5936         my ($id, $netstr) = ($1, $2);
5937         my $net = parse_net($netstr);
5938         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5939         $netstr = print_net($net);
5940         $res .= "$id: $netstr\n";
5941     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5942         my $virtdev = $1;
5943         my $value = $3;
5944         my $di = parse_drive($virtdev, $value);
5945         if (defined($di->{backup}) && !$di->{backup}) {
5946             $res .= "#$line";
5947         } elsif ($map->{$virtdev}) {
5948             delete $di->{format}; # format can change on restore
5949             $di->{file} = $map->{$virtdev};
5950             $value = print_drive($di);
5951             $res .= "$virtdev: $value\n";
5952         } else {
5953             $res .= $line;
5954         }
5955     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5956         my $vmgenid = $1;
5957         if ($vmgenid ne '0') {
5958             # always generate a new vmgenid if there was a valid one setup
5959             $vmgenid = generate_uuid();
5960         }
5961         $res .= "vmgenid: $vmgenid\n";
5962     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5963         my ($uuid, $uuid_str);
5964         UUID::generate($uuid);
5965         UUID::unparse($uuid, $uuid_str);
5966         my $smbios1 = parse_smbios1($2);
5967         $smbios1->{uuid} = $uuid_str;
5968         $res .= $1.print_smbios1($smbios1)."\n";
5969     } else {
5970         $res .= $line;
5971     }
5972
5973     return $res;
5974 };
5975
5976 my $restore_deactivate_volumes = sub {
5977     my ($storecfg, $devinfo) = @_;
5978
5979     my $vollist = [];
5980     foreach my $devname (keys %$devinfo) {
5981         my $volid = $devinfo->{$devname}->{volid};
5982         push @$vollist, $volid if $volid;
5983     }
5984
5985     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5986 };
5987
5988 my $restore_destroy_volumes = sub {
5989     my ($storecfg, $devinfo) = @_;
5990
5991     foreach my $devname (keys %$devinfo) {
5992         my $volid = $devinfo->{$devname}->{volid};
5993         next if !$volid;
5994         eval {
5995             if ($volid =~ m|^/|) {
5996                 unlink $volid || die 'unlink failed\n';
5997             } else {
5998                 PVE::Storage::vdisk_free($storecfg, $volid);
5999             }
6000             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6001         };
6002         print STDERR "unable to cleanup '$volid' - $@" if $@;
6003     }
6004 };
6005
6006 sub scan_volids {
6007     my ($cfg, $vmid) = @_;
6008
6009     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6010
6011     my $volid_hash = {};
6012     foreach my $storeid (keys %$info) {
6013         foreach my $item (@{$info->{$storeid}}) {
6014             next if !($item->{volid} && $item->{size});
6015             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6016             $volid_hash->{$item->{volid}} = $item;
6017         }
6018     }
6019
6020     return $volid_hash;
6021 }
6022
6023 sub update_disk_config {
6024     my ($vmid, $conf, $volid_hash) = @_;
6025
6026     my $changes;
6027     my $prefix = "VM $vmid";
6028
6029     # used and unused disks
6030     my $referenced = {};
6031
6032     # Note: it is allowed to define multiple storages with same path (alias), so
6033     # we need to check both 'volid' and real 'path' (two different volid can point
6034     # to the same path).
6035
6036     my $referencedpath = {};
6037
6038     # update size info
6039     PVE::QemuConfig->foreach_volume($conf, sub {
6040         my ($opt, $drive) = @_;
6041
6042         my $volid = $drive->{file};
6043         return if !$volid;
6044         my $volume = $volid_hash->{$volid};
6045
6046         # mark volid as "in-use" for next step
6047         $referenced->{$volid} = 1;
6048         if ($volume && (my $path = $volume->{path})) {
6049             $referencedpath->{$path} = 1;
6050         }
6051
6052         return if drive_is_cdrom($drive);
6053         return if !$volume;
6054
6055         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6056         if (defined($updated)) {
6057             $changes = 1;
6058             $conf->{$opt} = print_drive($updated);
6059             print "$prefix ($opt): $msg\n";
6060         }
6061     });
6062
6063     # remove 'unusedX' entry if volume is used
6064     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6065         my ($opt, $drive) = @_;
6066
6067         my $volid = $drive->{file};
6068         return if !$volid;
6069
6070         my $path;
6071         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6072         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6073             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6074             $changes = 1;
6075             delete $conf->{$opt};
6076         }
6077
6078         $referenced->{$volid} = 1;
6079         $referencedpath->{$path} = 1 if $path;
6080     });
6081
6082     foreach my $volid (sort keys %$volid_hash) {
6083         next if $volid =~ m/vm-$vmid-state-/;
6084         next if $referenced->{$volid};
6085         my $path = $volid_hash->{$volid}->{path};
6086         next if !$path; # just to be sure
6087         next if $referencedpath->{$path};
6088         $changes = 1;
6089         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6090         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6091         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6092     }
6093
6094     return $changes;
6095 }
6096
6097 sub rescan {
6098     my ($vmid, $nolock, $dryrun) = @_;
6099
6100     my $cfg = PVE::Storage::config();
6101
6102     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6103     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6104     foreach my $stor (keys %{$cfg->{ids}}) {
6105         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6106     }
6107
6108     print "rescan volumes...\n";
6109     my $volid_hash = scan_volids($cfg, $vmid);
6110
6111     my $updatefn =  sub {
6112         my ($vmid) = @_;
6113
6114         my $conf = PVE::QemuConfig->load_config($vmid);
6115
6116         PVE::QemuConfig->check_lock($conf);
6117
6118         my $vm_volids = {};
6119         foreach my $volid (keys %$volid_hash) {
6120             my $info = $volid_hash->{$volid};
6121             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6122         }
6123
6124         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6125
6126         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6127     };
6128
6129     if (defined($vmid)) {
6130         if ($nolock) {
6131             &$updatefn($vmid);
6132         } else {
6133             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6134         }
6135     } else {
6136         my $vmlist = config_list();
6137         foreach my $vmid (keys %$vmlist) {
6138             if ($nolock) {
6139                 &$updatefn($vmid);
6140             } else {
6141                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6142             }
6143         }
6144     }
6145 }
6146
6147 sub restore_proxmox_backup_archive {
6148     my ($archive, $vmid, $user, $options) = @_;
6149
6150     my $storecfg = PVE::Storage::config();
6151
6152     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6153     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6154
6155     my $fingerprint = $scfg->{fingerprint};
6156     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6157
6158     my $repo = PVE::PBSClient::get_repository($scfg);
6159
6160     # This is only used for `pbs-restore`!
6161     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6162     local $ENV{PBS_PASSWORD} = $password;
6163     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6164
6165     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6166         PVE::Storage::parse_volname($storecfg, $archive);
6167
6168     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6169
6170     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6171
6172     my $tmpdir = "/var/tmp/vzdumptmp$$";
6173     rmtree $tmpdir;
6174     mkpath $tmpdir;
6175
6176     my $conffile = PVE::QemuConfig->config_file($vmid);
6177      # disable interrupts (always do cleanups)
6178     local $SIG{INT} =
6179         local $SIG{TERM} =
6180         local $SIG{QUIT} =
6181         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6182
6183     # Note: $oldconf is undef if VM does not exists
6184     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6185     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6186     my $new_conf_raw = '';
6187
6188     my $rpcenv = PVE::RPCEnvironment::get();
6189     my $devinfo = {};
6190
6191     eval {
6192         # enable interrupts
6193         local $SIG{INT} =
6194             local $SIG{TERM} =
6195             local $SIG{QUIT} =
6196             local $SIG{HUP} =
6197             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6198
6199         my $cfgfn = "$tmpdir/qemu-server.conf";
6200         my $firewall_config_fn = "$tmpdir/fw.conf";
6201         my $index_fn = "$tmpdir/index.json";
6202
6203         my $cmd = "restore";
6204
6205         my $param = [$pbs_backup_name, "index.json", $index_fn];
6206         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6207         my $index = PVE::Tools::file_get_contents($index_fn);
6208         $index = decode_json($index);
6209
6210         # print Dumper($index);
6211         foreach my $info (@{$index->{files}}) {
6212             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6213                 my $devname = $1;
6214                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6215                     $devinfo->{$devname}->{size} = $1;
6216                 } else {
6217                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6218                 }
6219             }
6220         }
6221
6222         my $is_qemu_server_backup = scalar(
6223             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6224         );
6225         if (!$is_qemu_server_backup) {
6226             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6227         }
6228         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6229
6230         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6231         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6232
6233         if ($has_firewall_config) {
6234             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6235             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6236
6237             my $pve_firewall_dir = '/etc/pve/firewall';
6238             mkdir $pve_firewall_dir; # make sure the dir exists
6239             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6240         }
6241
6242         my $fh = IO::File->new($cfgfn, "r") ||
6243             die "unable to read qemu-server.conf - $!\n";
6244
6245         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6246
6247         # fixme: rate limit?
6248
6249         # create empty/temp config
6250         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6251
6252         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6253
6254         # allocate volumes
6255         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6256
6257         foreach my $virtdev (sort keys %$virtdev_hash) {
6258             my $d = $virtdev_hash->{$virtdev};
6259             next if $d->{is_cloudinit}; # no need to restore cloudinit
6260
6261             my $volid = $d->{volid};
6262
6263             my $path = PVE::Storage::path($storecfg, $volid);
6264
6265             # This is the ONLY user of the PBS_ env vars set on top of this function!
6266             my $pbs_restore_cmd = [
6267                 '/usr/bin/pbs-restore',
6268                 '--repository', $repo,
6269                 $pbs_backup_name,
6270                 "$d->{devname}.img.fidx",
6271                 $path,
6272                 '--verbose',
6273                 ];
6274
6275             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6276             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6277
6278             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6279                 push @$pbs_restore_cmd, '--skip-zero';
6280             }
6281
6282             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6283             print "restore proxmox backup image: $dbg_cmdstring\n";
6284             run_command($pbs_restore_cmd);
6285         }
6286
6287         $fh->seek(0, 0) || die "seek failed - $!\n";
6288
6289         my $cookie = { netcount => 0 };
6290         while (defined(my $line = <$fh>)) {
6291             $new_conf_raw .= $restore_update_config_line->(
6292                 $cookie,
6293                 $vmid,
6294                 $map,
6295                 $line,
6296                 $options->{unique},
6297             );
6298         }
6299
6300         $fh->close();
6301     };
6302     my $err = $@;
6303
6304     $restore_deactivate_volumes->($storecfg, $devinfo);
6305
6306     rmtree $tmpdir;
6307
6308     if ($err) {
6309         $restore_destroy_volumes->($storecfg, $devinfo);
6310         die $err;
6311     }
6312
6313     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6314
6315     PVE::Cluster::cfs_update(); # make sure we read new file
6316
6317     eval { rescan($vmid, 1); };
6318     warn $@ if $@;
6319 }
6320
6321 sub restore_vma_archive {
6322     my ($archive, $vmid, $user, $opts, $comp) = @_;
6323
6324     my $readfrom = $archive;
6325
6326     my $cfg = PVE::Storage::config();
6327     my $commands = [];
6328     my $bwlimit = $opts->{bwlimit};
6329
6330     my $dbg_cmdstring = '';
6331     my $add_pipe = sub {
6332         my ($cmd) = @_;
6333         push @$commands, $cmd;
6334         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6335         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6336         $readfrom = '-';
6337     };
6338
6339     my $input = undef;
6340     if ($archive eq '-') {
6341         $input = '<&STDIN';
6342     } else {
6343         # If we use a backup from a PVE defined storage we also consider that
6344         # storage's rate limit:
6345         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6346         if (defined($volid)) {
6347             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6348             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6349             if ($readlimit) {
6350                 print STDERR "applying read rate limit: $readlimit\n";
6351                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6352                 $add_pipe->($cstream);
6353             }
6354         }
6355     }
6356
6357     if ($comp) {
6358         my $info = PVE::Storage::decompressor_info('vma', $comp);
6359         my $cmd = $info->{decompressor};
6360         push @$cmd, $readfrom;
6361         $add_pipe->($cmd);
6362     }
6363
6364     my $tmpdir = "/var/tmp/vzdumptmp$$";
6365     rmtree $tmpdir;
6366
6367     # disable interrupts (always do cleanups)
6368     local $SIG{INT} =
6369         local $SIG{TERM} =
6370         local $SIG{QUIT} =
6371         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6372
6373     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6374     POSIX::mkfifo($mapfifo, 0600);
6375     my $fifofh;
6376     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6377
6378     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6379
6380     my $oldtimeout;
6381     my $timeout = 5;
6382
6383     my $devinfo = {};
6384
6385     my $rpcenv = PVE::RPCEnvironment::get();
6386
6387     my $conffile = PVE::QemuConfig->config_file($vmid);
6388
6389     # Note: $oldconf is undef if VM does not exist
6390     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6391     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6392     my $new_conf_raw = '';
6393
6394     my %storage_limits;
6395
6396     my $print_devmap = sub {
6397         my $cfgfn = "$tmpdir/qemu-server.conf";
6398
6399         # we can read the config - that is already extracted
6400         my $fh = IO::File->new($cfgfn, "r") ||
6401             die "unable to read qemu-server.conf - $!\n";
6402
6403         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6404         if (-f $fwcfgfn) {
6405             my $pve_firewall_dir = '/etc/pve/firewall';
6406             mkdir $pve_firewall_dir; # make sure the dir exists
6407             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6408         }
6409
6410         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6411
6412         foreach my $info (values %{$virtdev_hash}) {
6413             my $storeid = $info->{storeid};
6414             next if defined($storage_limits{$storeid});
6415
6416             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6417             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6418             $storage_limits{$storeid} = $limit * 1024;
6419         }
6420
6421         foreach my $devname (keys %$devinfo) {
6422             die "found no device mapping information for device '$devname'\n"
6423                 if !$devinfo->{$devname}->{virtdev};
6424         }
6425
6426         # create empty/temp config
6427         if ($oldconf) {
6428             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6429             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6430         }
6431
6432         # allocate volumes
6433         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6434
6435         # print restore information to $fifofh
6436         foreach my $virtdev (sort keys %$virtdev_hash) {
6437             my $d = $virtdev_hash->{$virtdev};
6438             next if $d->{is_cloudinit}; # no need to restore cloudinit
6439
6440             my $storeid = $d->{storeid};
6441             my $volid = $d->{volid};
6442
6443             my $map_opts = '';
6444             if (my $limit = $storage_limits{$storeid}) {
6445                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6446             }
6447
6448             my $write_zeros = 1;
6449             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6450                 $write_zeros = 0;
6451             }
6452
6453             my $path = PVE::Storage::path($cfg, $volid);
6454
6455             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6456
6457             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6458         }
6459
6460         $fh->seek(0, 0) || die "seek failed - $!\n";
6461
6462         my $cookie = { netcount => 0 };
6463         while (defined(my $line = <$fh>)) {
6464             $new_conf_raw .= $restore_update_config_line->(
6465                 $cookie,
6466                 $vmid,
6467                 $map,
6468                 $line,
6469                 $opts->{unique},
6470             );
6471         }
6472
6473         $fh->close();
6474     };
6475
6476     eval {
6477         # enable interrupts
6478         local $SIG{INT} =
6479             local $SIG{TERM} =
6480             local $SIG{QUIT} =
6481             local $SIG{HUP} =
6482             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6483         local $SIG{ALRM} = sub { die "got timeout\n"; };
6484
6485         $oldtimeout = alarm($timeout);
6486
6487         my $parser = sub {
6488             my $line = shift;
6489
6490             print "$line\n";
6491
6492             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6493                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6494                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6495             } elsif ($line =~ m/^CTIME: /) {
6496                 # we correctly received the vma config, so we can disable
6497                 # the timeout now for disk allocation (set to 10 minutes, so
6498                 # that we always timeout if something goes wrong)
6499                 alarm(600);
6500                 &$print_devmap();
6501                 print $fifofh "done\n";
6502                 my $tmp = $oldtimeout || 0;
6503                 $oldtimeout = undef;
6504                 alarm($tmp);
6505                 close($fifofh);
6506                 $fifofh = undef;
6507             }
6508         };
6509
6510         print "restore vma archive: $dbg_cmdstring\n";
6511         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6512     };
6513     my $err = $@;
6514
6515     alarm($oldtimeout) if $oldtimeout;
6516
6517     $restore_deactivate_volumes->($cfg, $devinfo);
6518
6519     close($fifofh) if $fifofh;
6520     unlink $mapfifo;
6521     rmtree $tmpdir;
6522
6523     if ($err) {
6524         $restore_destroy_volumes->($cfg, $devinfo);
6525         die $err;
6526     }
6527
6528     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6529
6530     PVE::Cluster::cfs_update(); # make sure we read new file
6531
6532     eval { rescan($vmid, 1); };
6533     warn $@ if $@;
6534 }
6535
6536 sub restore_tar_archive {
6537     my ($archive, $vmid, $user, $opts) = @_;
6538
6539     if ($archive ne '-') {
6540         my $firstfile = tar_archive_read_firstfile($archive);
6541         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6542             if $firstfile ne 'qemu-server.conf';
6543     }
6544
6545     my $storecfg = PVE::Storage::config();
6546
6547     # avoid zombie disks when restoring over an existing VM -> cleanup first
6548     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6549     # skiplock=1 because qmrestore has set the 'create' lock itself already
6550     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6551     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6552
6553     my $tocmd = "/usr/lib/qemu-server/qmextract";
6554
6555     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6556     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6557     $tocmd .= ' --prealloc' if $opts->{prealloc};
6558     $tocmd .= ' --info' if $opts->{info};
6559
6560     # tar option "xf" does not autodetect compression when read from STDIN,
6561     # so we pipe to zcat
6562     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6563         PVE::Tools::shellquote("--to-command=$tocmd");
6564
6565     my $tmpdir = "/var/tmp/vzdumptmp$$";
6566     mkpath $tmpdir;
6567
6568     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6569     local $ENV{VZDUMP_VMID} = $vmid;
6570     local $ENV{VZDUMP_USER} = $user;
6571
6572     my $conffile = PVE::QemuConfig->config_file($vmid);
6573     my $new_conf_raw = '';
6574
6575     # disable interrupts (always do cleanups)
6576     local $SIG{INT} =
6577         local $SIG{TERM} =
6578         local $SIG{QUIT} =
6579         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6580
6581     eval {
6582         # enable interrupts
6583         local $SIG{INT} =
6584             local $SIG{TERM} =
6585             local $SIG{QUIT} =
6586             local $SIG{HUP} =
6587             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6588
6589         if ($archive eq '-') {
6590             print "extracting archive from STDIN\n";
6591             run_command($cmd, input => "<&STDIN");
6592         } else {
6593             print "extracting archive '$archive'\n";
6594             run_command($cmd);
6595         }
6596
6597         return if $opts->{info};
6598
6599         # read new mapping
6600         my $map = {};
6601         my $statfile = "$tmpdir/qmrestore.stat";
6602         if (my $fd = IO::File->new($statfile, "r")) {
6603             while (defined (my $line = <$fd>)) {
6604                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6605                     $map->{$1} = $2 if $1;
6606                 } else {
6607                     print STDERR "unable to parse line in statfile - $line\n";
6608                 }
6609             }
6610             $fd->close();
6611         }
6612
6613         my $confsrc = "$tmpdir/qemu-server.conf";
6614
6615         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6616
6617         my $cookie = { netcount => 0 };
6618         while (defined (my $line = <$srcfd>)) {
6619             $new_conf_raw .= $restore_update_config_line->(
6620                 $cookie,
6621                 $vmid,
6622                 $map,
6623                 $line,
6624                 $opts->{unique},
6625             );
6626         }
6627
6628         $srcfd->close();
6629     };
6630     if (my $err = $@) {
6631         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6632         die $err;
6633     }
6634
6635     rmtree $tmpdir;
6636
6637     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6638
6639     PVE::Cluster::cfs_update(); # make sure we read new file
6640
6641     eval { rescan($vmid, 1); };
6642     warn $@ if $@;
6643 };
6644
6645 sub foreach_storage_used_by_vm {
6646     my ($conf, $func) = @_;
6647
6648     my $sidhash = {};
6649
6650     PVE::QemuConfig->foreach_volume($conf, sub {
6651         my ($ds, $drive) = @_;
6652         return if drive_is_cdrom($drive);
6653
6654         my $volid = $drive->{file};
6655
6656         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6657         $sidhash->{$sid} = $sid if $sid;
6658     });
6659
6660     foreach my $sid (sort keys %$sidhash) {
6661         &$func($sid);
6662     }
6663 }
6664
6665 my $qemu_snap_storage = {
6666     rbd => 1,
6667 };
6668 sub do_snapshots_with_qemu {
6669     my ($storecfg, $volid) = @_;
6670
6671     my $storage_name = PVE::Storage::parse_volume_id($volid);
6672     my $scfg = $storecfg->{ids}->{$storage_name};
6673     die "could not find storage '$storage_name'\n" if !defined($scfg);
6674
6675     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6676         return 1;
6677     }
6678
6679     if ($volid =~ m/\.(qcow2|qed)$/){
6680         return 1;
6681     }
6682
6683     return;
6684 }
6685
6686 sub qga_check_running {
6687     my ($vmid, $nowarn) = @_;
6688
6689     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6690     if ($@) {
6691         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6692         return 0;
6693     }
6694     return 1;
6695 }
6696
6697 sub template_create {
6698     my ($vmid, $conf, $disk) = @_;
6699
6700     my $storecfg = PVE::Storage::config();
6701
6702     PVE::QemuConfig->foreach_volume($conf, sub {
6703         my ($ds, $drive) = @_;
6704
6705         return if drive_is_cdrom($drive);
6706         return if $disk && $ds ne $disk;
6707
6708         my $volid = $drive->{file};
6709         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6710
6711         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6712         $drive->{file} = $voliddst;
6713         $conf->{$ds} = print_drive($drive);
6714         PVE::QemuConfig->write_config($vmid, $conf);
6715     });
6716 }
6717
6718 sub convert_iscsi_path {
6719     my ($path) = @_;
6720
6721     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6722         my $portal = $1;
6723         my $target = $2;
6724         my $lun = $3;
6725
6726         my $initiator_name = get_initiator_name();
6727
6728         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6729                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6730     }
6731
6732     die "cannot convert iscsi path '$path', unkown format\n";
6733 }
6734
6735 sub qemu_img_convert {
6736     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6737
6738     my $storecfg = PVE::Storage::config();
6739     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6740     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6741
6742     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6743
6744     my $cachemode;
6745     my $src_path;
6746     my $src_is_iscsi = 0;
6747     my $src_format;
6748
6749     if ($src_storeid) {
6750         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6751         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6752         $src_format = qemu_img_format($src_scfg, $src_volname);
6753         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6754         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6755         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6756     } elsif (-f $src_volid) {
6757         $src_path = $src_volid;
6758         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6759             $src_format = $1;
6760         }
6761     }
6762
6763     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6764
6765     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6766     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6767     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6768     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6769
6770     my $cmd = [];
6771     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6772     push @$cmd, '-l', "snapshot.name=$snapname"
6773         if $snapname && $src_format && $src_format eq "qcow2";
6774     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6775     push @$cmd, '-T', $cachemode if defined($cachemode);
6776
6777     if ($src_is_iscsi) {
6778         push @$cmd, '--image-opts';
6779         $src_path = convert_iscsi_path($src_path);
6780     } elsif ($src_format) {
6781         push @$cmd, '-f', $src_format;
6782     }
6783
6784     if ($dst_is_iscsi) {
6785         push @$cmd, '--target-image-opts';
6786         $dst_path = convert_iscsi_path($dst_path);
6787     } else {
6788         push @$cmd, '-O', $dst_format;
6789     }
6790
6791     push @$cmd, $src_path;
6792
6793     if (!$dst_is_iscsi && $is_zero_initialized) {
6794         push @$cmd, "zeroinit:$dst_path";
6795     } else {
6796         push @$cmd, $dst_path;
6797     }
6798
6799     my $parser = sub {
6800         my $line = shift;
6801         if($line =~ m/\((\S+)\/100\%\)/){
6802             my $percent = $1;
6803             my $transferred = int($size * $percent / 100);
6804             my $remaining = $size - $transferred;
6805
6806             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6807         }
6808
6809     };
6810
6811     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6812     my $err = $@;
6813     die "copy failed: $err" if $err;
6814 }
6815
6816 sub qemu_img_format {
6817     my ($scfg, $volname) = @_;
6818
6819     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6820         return $1;
6821     } else {
6822         return "raw";
6823     }
6824 }
6825
6826 sub qemu_drive_mirror {
6827     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6828
6829     $jobs = {} if !$jobs;
6830
6831     my $qemu_target;
6832     my $format;
6833     $jobs->{"drive-$drive"} = {};
6834
6835     if ($dst_volid =~ /^nbd:/) {
6836         $qemu_target = $dst_volid;
6837         $format = "nbd";
6838     } else {
6839         my $storecfg = PVE::Storage::config();
6840         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6841
6842         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6843
6844         $format = qemu_img_format($dst_scfg, $dst_volname);
6845
6846         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6847
6848         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6849     }
6850
6851     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6852     $opts->{format} = $format if $format;
6853
6854     if (defined($src_bitmap)) {
6855         $opts->{sync} = 'incremental';
6856         $opts->{bitmap} = $src_bitmap;
6857         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6858     }
6859
6860     if (defined($bwlimit)) {
6861         $opts->{speed} = $bwlimit * 1024;
6862         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6863     } else {
6864         print "drive mirror is starting for drive-$drive\n";
6865     }
6866
6867     # if a job already runs for this device we get an error, catch it for cleanup
6868     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6869     if (my $err = $@) {
6870         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6871         warn "$@\n" if $@;
6872         die "mirroring error: $err\n";
6873     }
6874
6875     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6876 }
6877
6878 # $completion can be either
6879 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6880 # 'cancel': wait until all jobs are ready, block-job-cancel them
6881 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6882 sub qemu_drive_mirror_monitor {
6883     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6884
6885     $completion //= 'complete';
6886
6887     eval {
6888         my $err_complete = 0;
6889
6890         while (1) {
6891             die "storage migration timed out\n" if $err_complete > 300;
6892
6893             my $stats = mon_cmd($vmid, "query-block-jobs");
6894
6895             my $running_mirror_jobs = {};
6896             foreach my $stat (@$stats) {
6897                 next if $stat->{type} ne 'mirror';
6898                 $running_mirror_jobs->{$stat->{device}} = $stat;
6899             }
6900
6901             my $readycounter = 0;
6902
6903             foreach my $job (keys %$jobs) {
6904
6905                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6906                     print "$job : finished\n";
6907                     delete $jobs->{$job};
6908                     next;
6909                 }
6910
6911                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6912
6913                 my $busy = $running_mirror_jobs->{$job}->{busy};
6914                 my $ready = $running_mirror_jobs->{$job}->{ready};
6915                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6916                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6917                     my $remaining = $total - $transferred;
6918                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6919
6920                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6921                 }
6922
6923                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6924             }
6925
6926             last if scalar(keys %$jobs) == 0;
6927
6928             if ($readycounter == scalar(keys %$jobs)) {
6929                 print "all mirroring jobs are ready \n";
6930                 last if $completion eq 'skip'; #do the complete later
6931
6932                 if ($vmiddst && $vmiddst != $vmid) {
6933                     my $agent_running = $qga && qga_check_running($vmid);
6934                     if ($agent_running) {
6935                         print "freeze filesystem\n";
6936                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6937                     } else {
6938                         print "suspend vm\n";
6939                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6940                     }
6941
6942                     # if we clone a disk for a new target vm, we don't switch the disk
6943                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6944
6945                     if ($agent_running) {
6946                         print "unfreeze filesystem\n";
6947                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6948                     } else {
6949                         print "resume vm\n";
6950                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6951                     }
6952
6953                     last;
6954                 } else {
6955
6956                     foreach my $job (keys %$jobs) {
6957                         # try to switch the disk if source and destination are on the same guest
6958                         print "$job: Completing block job...\n";
6959
6960                         my $op;
6961                         if ($completion eq 'complete') {
6962                             $op = 'block-job-complete';
6963                         } elsif ($completion eq 'cancel') {
6964                             $op = 'block-job-cancel';
6965                         } else {
6966                             die "invalid completion value: $completion\n";
6967                         }
6968                         eval { mon_cmd($vmid, $op, device => $job) };
6969                         if ($@ =~ m/cannot be completed/) {
6970                             print "$job: Block job cannot be completed, try again.\n";
6971                             $err_complete++;
6972                         }else {
6973                             print "$job: Completed successfully.\n";
6974                             $jobs->{$job}->{complete} = 1;
6975                         }
6976                     }
6977                 }
6978             }
6979             sleep 1;
6980         }
6981     };
6982     my $err = $@;
6983
6984     if ($err) {
6985         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6986         die "mirroring error: $err";
6987     }
6988
6989 }
6990
6991 sub qemu_blockjobs_cancel {
6992     my ($vmid, $jobs) = @_;
6993
6994     foreach my $job (keys %$jobs) {
6995         print "$job: Cancelling block job\n";
6996         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6997         $jobs->{$job}->{cancel} = 1;
6998     }
6999
7000     while (1) {
7001         my $stats = mon_cmd($vmid, "query-block-jobs");
7002
7003         my $running_jobs = {};
7004         foreach my $stat (@$stats) {
7005             $running_jobs->{$stat->{device}} = $stat;
7006         }
7007
7008         foreach my $job (keys %$jobs) {
7009
7010             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7011                 print "$job: Done.\n";
7012                 delete $jobs->{$job};
7013             }
7014         }
7015
7016         last if scalar(keys %$jobs) == 0;
7017
7018         sleep 1;
7019     }
7020 }
7021
7022 sub clone_disk {
7023     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7024         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7025
7026     my $newvolid;
7027
7028     if (!$full) {
7029         print "create linked clone of drive $drivename ($drive->{file})\n";
7030         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7031         push @$newvollist, $newvolid;
7032     } else {
7033
7034         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7035         $storeid = $storage if $storage;
7036
7037         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7038
7039         print "create full clone of drive $drivename ($drive->{file})\n";
7040         my $name = undef;
7041         my $size = undef;
7042         if (drive_is_cloudinit($drive)) {
7043             $name = "vm-$newvmid-cloudinit";
7044             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7045             if ($scfg->{path}) {
7046                 $name .= ".$dst_format";
7047             }
7048             $snapname = undef;
7049             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7050         } elsif ($drivename eq 'efidisk0') {
7051             $size = get_efivars_size($conf);
7052         } else {
7053             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7054         }
7055         $newvolid = PVE::Storage::vdisk_alloc(
7056             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7057         );
7058         push @$newvollist, $newvolid;
7059
7060         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7061
7062         if (drive_is_cloudinit($drive)) {
7063             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7064             # if this is the case, we have to complete any block-jobs still there from
7065             # previous drive-mirrors
7066             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7067                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7068             }
7069             goto no_data_clone;
7070         }
7071
7072         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7073         if (!$running || $snapname) {
7074             # TODO: handle bwlimits
7075             if ($drivename eq 'efidisk0') {
7076                 # the relevant data on the efidisk may be smaller than the source
7077                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7078                 # that is given by the OVMF_VARS.fd
7079                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7080                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7081
7082                 # better for Ceph if block size is not too small, see bug #3324
7083                 my $bs = 1024*1024;
7084
7085                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7086                     "if=$src_path", "of=$dst_path"]);
7087             } else {
7088                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7089             }
7090         } else {
7091
7092             my $kvmver = get_running_qemu_version ($vmid);
7093             if (!min_version($kvmver, 2, 7)) {
7094                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7095                     if $drive->{iothread};
7096             }
7097
7098             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7099                 $completion, $qga, $bwlimit);
7100         }
7101     }
7102
7103 no_data_clone:
7104     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7105
7106     my $disk = $drive;
7107     $disk->{format} = undef;
7108     $disk->{file} = $newvolid;
7109     $disk->{size} = $size if defined($size);
7110
7111     return $disk;
7112 }
7113
7114 sub get_running_qemu_version {
7115     my ($vmid) = @_;
7116     my $res = mon_cmd($vmid, "query-version");
7117     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7118 }
7119
7120 sub qemu_use_old_bios_files {
7121     my ($machine_type) = @_;
7122
7123     return if !$machine_type;
7124
7125     my $use_old_bios_files = undef;
7126
7127     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7128         $machine_type = $1;
7129         $use_old_bios_files = 1;
7130     } else {
7131         my $version = extract_version($machine_type, kvm_user_version());
7132         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7133         # load new efi bios files on migration. So this hack is required to allow
7134         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7135         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7136         $use_old_bios_files = !min_version($version, 2, 4);
7137     }
7138
7139     return ($use_old_bios_files, $machine_type);
7140 }
7141
7142 sub get_efivars_size {
7143     my ($conf) = @_;
7144     my $arch = get_vm_arch($conf);
7145     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7146     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7147     return -s $ovmf_vars;
7148 }
7149
7150 sub update_efidisk_size {
7151     my ($conf) = @_;
7152
7153     return if !defined($conf->{efidisk0});
7154
7155     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7156     $disk->{size} = get_efivars_size($conf);
7157     $conf->{efidisk0} = print_drive($disk);
7158
7159     return;
7160 }
7161
7162 sub create_efidisk($$$$$) {
7163     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7164
7165     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7166     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7167
7168     my $vars_size_b = -s $ovmf_vars;
7169     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7170     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7171     PVE::Storage::activate_volumes($storecfg, [$volid]);
7172
7173     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7174     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7175
7176     return ($volid, $size/1024);
7177 }
7178
7179 sub vm_iothreads_list {
7180     my ($vmid) = @_;
7181
7182     my $res = mon_cmd($vmid, 'query-iothreads');
7183
7184     my $iothreads = {};
7185     foreach my $iothread (@$res) {
7186         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7187     }
7188
7189     return $iothreads;
7190 }
7191
7192 sub scsihw_infos {
7193     my ($conf, $drive) = @_;
7194
7195     my $maxdev = 0;
7196
7197     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7198         $maxdev = 7;
7199     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7200         $maxdev = 1;
7201     } else {
7202         $maxdev = 256;
7203     }
7204
7205     my $controller = int($drive->{index} / $maxdev);
7206     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7207         ? "virtioscsi"
7208         : "scsihw";
7209
7210     return ($maxdev, $controller, $controller_prefix);
7211 }
7212
7213 sub windows_version {
7214     my ($ostype) = @_;
7215
7216     return 0 if !$ostype;
7217
7218     my $winversion = 0;
7219
7220     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7221         $winversion = 5;
7222     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7223         $winversion = 6;
7224     } elsif ($ostype =~ m/^win(\d+)$/) {
7225         $winversion = $1;
7226     }
7227
7228     return $winversion;
7229 }
7230
7231 sub resolve_dst_disk_format {
7232         my ($storecfg, $storeid, $src_volname, $format) = @_;
7233         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7234
7235         if (!$format) {
7236             # if no target format is specified, use the source disk format as hint
7237             if ($src_volname) {
7238                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7239                 $format = qemu_img_format($scfg, $src_volname);
7240             } else {
7241                 return $defFormat;
7242             }
7243         }
7244
7245         # test if requested format is supported - else use default
7246         my $supported = grep { $_ eq $format } @$validFormats;
7247         $format = $defFormat if !$supported;
7248         return $format;
7249 }
7250
7251 # NOTE: if this logic changes, please update docs & possibly gui logic
7252 sub find_vmstate_storage {
7253     my ($conf, $storecfg) = @_;
7254
7255     # first, return storage from conf if set
7256     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7257
7258     my ($target, $shared, $local);
7259
7260     foreach_storage_used_by_vm($conf, sub {
7261         my ($sid) = @_;
7262         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7263         my $dst = $scfg->{shared} ? \$shared : \$local;
7264         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7265     });
7266
7267     # second, use shared storage where VM has at least one disk
7268     # third, use local storage where VM has at least one disk
7269     # fall back to local storage
7270     $target = $shared // $local // 'local';
7271
7272     return $target;
7273 }
7274
7275 sub generate_uuid {
7276     my ($uuid, $uuid_str);
7277     UUID::generate($uuid);
7278     UUID::unparse($uuid, $uuid_str);
7279     return $uuid_str;
7280 }
7281
7282 sub generate_smbios1_uuid {
7283     return "uuid=".generate_uuid();
7284 }
7285
7286 sub nbd_stop {
7287     my ($vmid) = @_;
7288
7289     mon_cmd($vmid, 'nbd-server-stop');
7290 }
7291
7292 sub create_reboot_request {
7293     my ($vmid) = @_;
7294     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7295         or die "failed to create reboot trigger file: $!\n";
7296     close($fh);
7297 }
7298
7299 sub clear_reboot_request {
7300     my ($vmid) = @_;
7301     my $path = "/run/qemu-server/$vmid.reboot";
7302     my $res = 0;
7303
7304     $res = unlink($path);
7305     die "could not remove reboot request for $vmid: $!"
7306         if !$res && $! != POSIX::ENOENT;
7307
7308     return $res;
7309 }
7310
7311 sub bootorder_from_legacy {
7312     my ($conf, $bootcfg) = @_;
7313
7314     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7315     my $bootindex_hash = {};
7316     my $i = 1;
7317     foreach my $o (split(//, $boot)) {
7318         $bootindex_hash->{$o} = $i*100;
7319         $i++;
7320     }
7321
7322     my $bootorder = {};
7323
7324     PVE::QemuConfig->foreach_volume($conf, sub {
7325         my ($ds, $drive) = @_;
7326
7327         if (drive_is_cdrom ($drive, 1)) {
7328             if ($bootindex_hash->{d}) {
7329                 $bootorder->{$ds} = $bootindex_hash->{d};
7330                 $bootindex_hash->{d} += 1;
7331             }
7332         } elsif ($bootindex_hash->{c}) {
7333             $bootorder->{$ds} = $bootindex_hash->{c}
7334                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7335             $bootindex_hash->{c} += 1;
7336         }
7337     });
7338
7339     if ($bootindex_hash->{n}) {
7340         for (my $i = 0; $i < $MAX_NETS; $i++) {
7341             my $netname = "net$i";
7342             next if !$conf->{$netname};
7343             $bootorder->{$netname} = $bootindex_hash->{n};
7344             $bootindex_hash->{n} += 1;
7345         }
7346     }
7347
7348     return $bootorder;
7349 }
7350
7351 # Generate default device list for 'boot: order=' property. Matches legacy
7352 # default boot order, but with explicit device names. This is important, since
7353 # the fallback for when neither 'order' nor the old format is specified relies
7354 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7355 sub get_default_bootdevices {
7356     my ($conf) = @_;
7357
7358     my @ret = ();
7359
7360     # harddisk
7361     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7362     push @ret, $first if $first;
7363
7364     # cdrom
7365     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7366     push @ret, $first if $first;
7367
7368     # network
7369     for (my $i = 0; $i < $MAX_NETS; $i++) {
7370         my $netname = "net$i";
7371         next if !$conf->{$netname};
7372         push @ret, $netname;
7373         last;
7374     }
7375
7376     return \@ret;
7377 }
7378
7379 sub device_bootorder {
7380     my ($conf) = @_;
7381
7382     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7383
7384     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7385
7386     my $bootorder = {};
7387     if (!defined($boot) || $boot->{legacy}) {
7388         $bootorder = bootorder_from_legacy($conf, $boot);
7389     } elsif ($boot->{order}) {
7390         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7391         for my $dev (PVE::Tools::split_list($boot->{order})) {
7392             $bootorder->{$dev} = $i++;
7393         }
7394     }
7395
7396     return $bootorder;
7397 }
7398
7399 # bash completion helper
7400
7401 sub complete_backup_archives {
7402     my ($cmdname, $pname, $cvalue) = @_;
7403
7404     my $cfg = PVE::Storage::config();
7405
7406     my $storeid;
7407
7408     if ($cvalue =~ m/^([^:]+):/) {
7409         $storeid = $1;
7410     }
7411
7412     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7413
7414     my $res = [];
7415     foreach my $id (keys %$data) {
7416         foreach my $item (@{$data->{$id}}) {
7417             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7418             push @$res, $item->{volid} if defined($item->{volid});
7419         }
7420     }
7421
7422     return $res;
7423 }
7424
7425 my $complete_vmid_full = sub {
7426     my ($running) = @_;
7427
7428     my $idlist = vmstatus();
7429
7430     my $res = [];
7431
7432     foreach my $id (keys %$idlist) {
7433         my $d = $idlist->{$id};
7434         if (defined($running)) {
7435             next if $d->{template};
7436             next if $running && $d->{status} ne 'running';
7437             next if !$running && $d->{status} eq 'running';
7438         }
7439         push @$res, $id;
7440
7441     }
7442     return $res;
7443 };
7444
7445 sub complete_vmid {
7446     return &$complete_vmid_full();
7447 }
7448
7449 sub complete_vmid_stopped {
7450     return &$complete_vmid_full(0);
7451 }
7452
7453 sub complete_vmid_running {
7454     return &$complete_vmid_full(1);
7455 }
7456
7457 sub complete_storage {
7458
7459     my $cfg = PVE::Storage::config();
7460     my $ids = $cfg->{ids};
7461
7462     my $res = [];
7463     foreach my $sid (keys %$ids) {
7464         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7465         next if !$ids->{$sid}->{content}->{images};
7466         push @$res, $sid;
7467     }
7468
7469     return $res;
7470 }
7471
7472 sub complete_migration_storage {
7473     my ($cmd, $param, $current_value, $all_args) = @_;
7474
7475     my $targetnode = @$all_args[1];
7476
7477     my $cfg = PVE::Storage::config();
7478     my $ids = $cfg->{ids};
7479
7480     my $res = [];
7481     foreach my $sid (keys %$ids) {
7482         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7483         next if !$ids->{$sid}->{content}->{images};
7484         push @$res, $sid;
7485     }
7486
7487     return $res;
7488 }
7489
7490 sub vm_is_paused {
7491     my ($vmid) = @_;
7492     my $qmpstatus = eval {
7493         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7494         mon_cmd($vmid, "query-status");
7495     };
7496     warn "$@\n" if $@;
7497     return $qmpstatus && $qmpstatus->{status} eq "paused";
7498 }
7499
7500 1;