PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016/2019
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud', 'opennebula'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 968 cloud-init 19.4 or newer.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 971 dhcp on IPv4.
 972 EODESCR
 973 };
 974 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 975
 976 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 977     $confdesc->{"net$i"} = $netdesc;
 978     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 979 }
 980
 981 foreach my $key (keys %$confdesc_cloudinit) {
 982     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 983 }
 984
 985 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 986 sub verify_volume_id_or_qm_path {
 987     my ($volid, $noerr) = @_;
 988
 989     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 990         return $volid;
 991     }
 992
 993     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 994     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 995     if ($@) {
 996         return if $noerr;
 997         die $@;
 998     }
 999     return $volid;
1000 }
1001
1002 my $usb_fmt = {
1003     host => {
1004         default_key => 1,
1005         type => 'string', format => 'pve-qm-usb-device',
1006         format_description => 'HOSTUSBDEVICE|spice',
1007         description => <<EODESCR,
1008 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1009
1010  'bus-port(.port)*' (decimal numbers) or
1011  'vendor_id:product_id' (hexadeciaml numbers) or
1012  'spice'
1013
1014 You can use the 'lsusb -t' command to list existing usb devices.
1015
1016 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1017 machines - use with special care.
1018
1019 The value 'spice' can be used to add a usb redirection devices for spice.
1020 EODESCR
1021     },
1022     usb3 => {
1023         optional => 1,
1024         type => 'boolean',
1025         description => "Specifies whether if given host option is a USB3 device or port.",
1026         default => 0,
1027     },
1028 };
1029
1030 my $usbdesc = {
1031     optional => 1,
1032     type => 'string', format => $usb_fmt,
1033     description => "Configure an USB device (n is 0 to 4).",
1034 };
1035 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1036
1037 my $serialdesc = {
1038         optional => 1,
1039         type => 'string',
1040         pattern => '(/dev/.+|socket)',
1041         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1042         verbose_description =>  <<EODESCR,
1043 Create a serial device inside the VM (n is 0 to 3), and pass through a
1044 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1045 host side (use 'qm terminal' to open a terminal connection).
1046
1047 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1048 use with special care.
1049
1050 CAUTION: Experimental! User reported problems with this option.
1051 EODESCR
1052 };
1053
1054 my $paralleldesc= {
1055         optional => 1,
1056         type => 'string',
1057         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1058         description =>  "Map host parallel devices (n is 0 to 2).",
1059         verbose_description =>  <<EODESCR,
1060 Map host parallel devices (n is 0 to 2).
1061
1062 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1063 machines - use with special care.
1064
1065 CAUTION: Experimental! User reported problems with this option.
1066 EODESCR
1067 };
1068
1069 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1070     $confdesc->{"parallel$i"} = $paralleldesc;
1071 }
1072
1073 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1074     $confdesc->{"serial$i"} = $serialdesc;
1075 }
1076
1077 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1078     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1079 }
1080
1081 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1082     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1083 }
1084
1085 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1086     $confdesc->{"usb$i"} = $usbdesc;
1087 }
1088
1089 my $boot_fmt = {
1090     legacy => {
1091         optional => 1,
1092         default_key => 1,
1093         type => 'string',
1094         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1095                      . " Deprecated, use 'order=' instead.",
1096         pattern => '[acdn]{1,4}',
1097         format_description => "[acdn]{1,4}",
1098
1099         # note: this is also the fallback if boot: is not given at all
1100         default => 'cdn',
1101     },
1102     order => {
1103         optional => 1,
1104         type => 'string',
1105         format => 'pve-qm-bootdev-list',
1106         format_description => "device[;device...]",
1107         description => <<EODESC,
1108 The guest will attempt to boot from devices in the order they appear here.
1109
1110 Disks, optical drives and passed-through storage USB devices will be directly
1111 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1112 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1113
1114 Note that only devices in this list will be marked as bootable and thus loaded
1115 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1116 (e.g. software-raid), you need to specify all of them here.
1117
1118 Overrides the deprecated 'legacy=[acdn]*' value when given.
1119 EODESC
1120     },
1121 };
1122 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1123
1124 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1125 sub verify_bootdev {
1126     my ($dev, $noerr) = @_;
1127
1128     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1129
1130     my $check = sub {
1131         my ($base) = @_;
1132         return 0 if $dev !~ m/^$base\d+$/;
1133         return 0 if !$confdesc->{$dev};
1134         return 1;
1135     };
1136
1137     return $dev if $check->("net");
1138     return $dev if $check->("usb");
1139     return $dev if $check->("hostpci");
1140
1141     return if $noerr;
1142     die "invalid boot device '$dev'\n";
1143 }
1144
1145 sub print_bootorder {
1146     my ($devs) = @_;
1147     return "" if !@$devs;
1148     my $data = { order => join(';', @$devs) };
1149     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1150 }
1151
1152 my $kvm_api_version = 0;
1153
1154 sub kvm_version {
1155     return $kvm_api_version if $kvm_api_version;
1156
1157     open my $fh, '<', '/dev/kvm' or return;
1158
1159     # 0xae00 => KVM_GET_API_VERSION
1160     $kvm_api_version = ioctl($fh, 0xae00, 0);
1161     close($fh);
1162
1163     return $kvm_api_version;
1164 }
1165
1166 my $kvm_user_version = {};
1167 my $kvm_mtime = {};
1168
1169 sub kvm_user_version {
1170     my ($binary) = @_;
1171
1172     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1173     my $st = stat($binary);
1174
1175     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1176     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1177         $cachedmtime == $st->mtime;
1178
1179     $kvm_user_version->{$binary} = 'unknown';
1180     $kvm_mtime->{$binary} = $st->mtime;
1181
1182     my $code = sub {
1183         my $line = shift;
1184         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1185             $kvm_user_version->{$binary} = $2;
1186         }
1187     };
1188
1189     eval { run_command([$binary, '--version'], outfunc => $code); };
1190     warn $@ if $@;
1191
1192     return $kvm_user_version->{$binary};
1193
1194 }
1195 my sub extract_version {
1196     my ($machine_type, $version) = @_;
1197     $version = kvm_user_version() if !defined($version);
1198     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1199 }
1200
1201 sub kernel_has_vhost_net {
1202     return -c '/dev/vhost-net';
1203 }
1204
1205 sub option_exists {
1206     my $key = shift;
1207     return defined($confdesc->{$key});
1208 }
1209
1210 my $cdrom_path;
1211 sub get_cdrom_path {
1212
1213     return  $cdrom_path if $cdrom_path;
1214
1215     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1216     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1217     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1218 }
1219
1220 sub get_iso_path {
1221     my ($storecfg, $vmid, $cdrom) = @_;
1222
1223     if ($cdrom eq 'cdrom') {
1224         return get_cdrom_path();
1225     } elsif ($cdrom eq 'none') {
1226         return '';
1227     } elsif ($cdrom =~ m|^/|) {
1228         return $cdrom;
1229     } else {
1230         return PVE::Storage::path($storecfg, $cdrom);
1231     }
1232 }
1233
1234 # try to convert old style file names to volume IDs
1235 sub filename_to_volume_id {
1236     my ($vmid, $file, $media) = @_;
1237
1238      if (!($file eq 'none' || $file eq 'cdrom' ||
1239           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1240
1241         return if $file =~ m|/|;
1242
1243         if ($media && $media eq 'cdrom') {
1244             $file = "local:iso/$file";
1245         } else {
1246             $file = "local:$vmid/$file";
1247         }
1248     }
1249
1250     return $file;
1251 }
1252
1253 sub verify_media_type {
1254     my ($opt, $vtype, $media) = @_;
1255
1256     return if !$media;
1257
1258     my $etype;
1259     if ($media eq 'disk') {
1260         $etype = 'images';
1261     } elsif ($media eq 'cdrom') {
1262         $etype = 'iso';
1263     } else {
1264         die "internal error";
1265     }
1266
1267     return if ($vtype eq $etype);
1268
1269     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1270 }
1271
1272 sub cleanup_drive_path {
1273     my ($opt, $storecfg, $drive) = @_;
1274
1275     # try to convert filesystem paths to volume IDs
1276
1277     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1278         ($drive->{file} !~ m|^/dev/.+|) &&
1279         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1280         ($drive->{file} !~ m/^\d+$/)) {
1281         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1282         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1283             if !$vtype;
1284         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1285         verify_media_type($opt, $vtype, $drive->{media});
1286         $drive->{file} = $volid;
1287     }
1288
1289     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1290 }
1291
1292 sub parse_hotplug_features {
1293     my ($data) = @_;
1294
1295     my $res = {};
1296
1297     return $res if $data eq '0';
1298
1299     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1300
1301     foreach my $feature (PVE::Tools::split_list($data)) {
1302         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1303             $res->{$1} = 1;
1304         } else {
1305             die "invalid hotplug feature '$feature'\n";
1306         }
1307     }
1308     return $res;
1309 }
1310
1311 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1312 sub pve_verify_hotplug_features {
1313     my ($value, $noerr) = @_;
1314
1315     return $value if parse_hotplug_features($value);
1316
1317     return if $noerr;
1318
1319     die "unable to parse hotplug option\n";
1320 }
1321
1322 sub scsi_inquiry {
1323     my($fh, $noerr) = @_;
1324
1325     my $SG_IO = 0x2285;
1326     my $SG_GET_VERSION_NUM = 0x2282;
1327
1328     my $versionbuf = "\x00" x 8;
1329     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1330     if (!$ret) {
1331         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1332         return;
1333     }
1334     my $version = unpack("I", $versionbuf);
1335     if ($version < 30000) {
1336         die "scsi generic interface too old\n"  if !$noerr;
1337         return;
1338     }
1339
1340     my $buf = "\x00" x 36;
1341     my $sensebuf = "\x00" x 8;
1342     my $cmd = pack("C x3 C x1", 0x12, 36);
1343
1344     # see /usr/include/scsi/sg.h
1345     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1346
1347     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1348                       length($sensebuf), 0, length($buf), $buf,
1349                       $cmd, $sensebuf, 6000);
1350
1351     $ret = ioctl($fh, $SG_IO, $packet);
1352     if (!$ret) {
1353         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1354         return;
1355     }
1356
1357     my @res = unpack($sg_io_hdr_t, $packet);
1358     if ($res[17] || $res[18]) {
1359         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1360         return;
1361     }
1362
1363     my $res = {};
1364     (my $byte0, my $byte1, $res->{vendor},
1365      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1366
1367     $res->{removable} = $byte1 & 128 ? 1 : 0;
1368     $res->{type} = $byte0 & 31;
1369
1370     return $res;
1371 }
1372
1373 sub path_is_scsi {
1374     my ($path) = @_;
1375
1376     my $fh = IO::File->new("+<$path") || return;
1377     my $res = scsi_inquiry($fh, 1);
1378     close($fh);
1379
1380     return $res;
1381 }
1382
1383 sub print_tabletdevice_full {
1384     my ($conf, $arch) = @_;
1385
1386     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1387
1388     # we use uhci for old VMs because tablet driver was buggy in older qemu
1389     my $usbbus;
1390     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1391         $usbbus = 'ehci';
1392     } else {
1393         $usbbus = 'uhci';
1394     }
1395
1396     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1397 }
1398
1399 sub print_keyboarddevice_full {
1400     my ($conf, $arch, $machine) = @_;
1401
1402     return if $arch ne 'aarch64';
1403
1404     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1405 }
1406
1407 sub print_drivedevice_full {
1408     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1409
1410     my $device = '';
1411     my $maxdev = 0;
1412
1413     my $drive_id = "$drive->{interface}$drive->{index}";
1414     if ($drive->{interface} eq 'virtio') {
1415         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1416         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1417         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1418     } elsif ($drive->{interface} eq 'scsi') {
1419
1420         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1421         my $unit = $drive->{index} % $maxdev;
1422         my $devicetype = 'hd';
1423         my $path = '';
1424         if (drive_is_cdrom($drive)) {
1425             $devicetype = 'cd';
1426         } else {
1427             if ($drive->{file} =~ m|^/|) {
1428                 $path = $drive->{file};
1429                 if (my $info = path_is_scsi($path)) {
1430                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1431                         $devicetype = 'block';
1432                     } elsif ($info->{type} == 1) { # tape
1433                         $devicetype = 'generic';
1434                     }
1435                 }
1436             } else {
1437                  $path = PVE::Storage::path($storecfg, $drive->{file});
1438             }
1439
1440             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1441             my $version = extract_version($machine_type, kvm_user_version());
1442             if ($path =~ m/^iscsi\:\/\// &&
1443                !min_version($version, 4, 1)) {
1444                 $devicetype = 'generic';
1445             }
1446         }
1447
1448         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1449             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1450         } else {
1451             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1452                 .",lun=$drive->{index}";
1453         }
1454         $device .= ",drive=drive-$drive_id,id=$drive_id";
1455
1456         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1457             $device .= ",rotation_rate=1";
1458         }
1459         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1460
1461     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1462         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1463         my $controller = int($drive->{index} / $maxdev);
1464         my $unit = $drive->{index} % $maxdev;
1465         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1466
1467         $device = "ide-$devicetype";
1468         if ($drive->{interface} eq 'ide') {
1469             $device .= ",bus=ide.$controller,unit=$unit";
1470         } else {
1471             $device .= ",bus=ahci$controller.$unit";
1472         }
1473         $device .= ",drive=drive-$drive_id,id=$drive_id";
1474
1475         if ($devicetype eq 'hd') {
1476             if (my $model = $drive->{model}) {
1477                 $model = URI::Escape::uri_unescape($model);
1478                 $device .= ",model=$model";
1479             }
1480             if ($drive->{ssd}) {
1481                 $device .= ",rotation_rate=1";
1482             }
1483         }
1484         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1485     } elsif ($drive->{interface} eq 'usb') {
1486         die "implement me";
1487         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1488     } else {
1489         die "unsupported interface type";
1490     }
1491
1492     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1493
1494     if (my $serial = $drive->{serial}) {
1495         $serial = URI::Escape::uri_unescape($serial);
1496         $device .= ",serial=$serial";
1497     }
1498
1499
1500     return $device;
1501 }
1502
1503 sub get_initiator_name {
1504     my $initiator;
1505
1506     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1507     while (defined(my $line = <$fh>)) {
1508         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1509         $initiator = $1;
1510         last;
1511     }
1512     $fh->close();
1513
1514     return $initiator;
1515 }
1516
1517 sub print_drive_commandline_full {
1518     my ($storecfg, $vmid, $drive) = @_;
1519
1520     my $path;
1521     my $volid = $drive->{file};
1522     my $format;
1523
1524     if (drive_is_cdrom($drive)) {
1525         $path = get_iso_path($storecfg, $vmid, $volid);
1526     } else {
1527         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1528         if ($storeid) {
1529             $path = PVE::Storage::path($storecfg, $volid);
1530             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1531             $format = qemu_img_format($scfg, $volname);
1532         } else {
1533             $path = $volid;
1534             $format = "raw";
1535         }
1536    }
1537
1538     my $opts = '';
1539     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1540     foreach my $o (@qemu_drive_options) {
1541         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1542     }
1543
1544     # snapshot only accepts on|off
1545     if (defined($drive->{snapshot})) {
1546         my $v = $drive->{snapshot} ? 'on' : 'off';
1547         $opts .= ",snapshot=$v";
1548     }
1549
1550     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1551         my ($dir, $qmpname) = @$type;
1552         if (my $v = $drive->{"mbps$dir"}) {
1553             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1554         }
1555         if (my $v = $drive->{"mbps${dir}_max"}) {
1556             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1557         }
1558         if (my $v = $drive->{"bps${dir}_max_length"}) {
1559             $opts .= ",throttling.bps$qmpname-max-length=$v";
1560         }
1561         if (my $v = $drive->{"iops${dir}"}) {
1562             $opts .= ",throttling.iops$qmpname=$v";
1563         }
1564         if (my $v = $drive->{"iops${dir}_max"}) {
1565             $opts .= ",throttling.iops$qmpname-max=$v";
1566         }
1567         if (my $v = $drive->{"iops${dir}_max_length"}) {
1568             $opts .= ",throttling.iops$qmpname-max-length=$v";
1569         }
1570     }
1571
1572     $opts .= ",format=$format" if $format && !$drive->{format};
1573
1574     my $cache_direct = 0;
1575
1576     if (my $cache = $drive->{cache}) {
1577         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1578     } elsif (!drive_is_cdrom($drive)) {
1579         $opts .= ",cache=none";
1580         $cache_direct = 1;
1581     }
1582
1583     # aio native works only with O_DIRECT
1584     if (!$drive->{aio}) {
1585         if($cache_direct) {
1586             $opts .= ",aio=native";
1587         } else {
1588             $opts .= ",aio=threads";
1589         }
1590     }
1591
1592     if (!drive_is_cdrom($drive)) {
1593         my $detectzeroes;
1594         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1595             $detectzeroes = 'off';
1596         } elsif ($drive->{discard}) {
1597             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1598         } else {
1599             # This used to be our default with discard not being specified:
1600             $detectzeroes = 'on';
1601         }
1602         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1603     }
1604
1605     my $pathinfo = $path ? "file=$path," : '';
1606
1607     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1608 }
1609
1610 sub print_netdevice_full {
1611     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1612
1613     my $device = $net->{model};
1614     if ($net->{model} eq 'virtio') {
1615          $device = 'virtio-net-pci';
1616      };
1617
1618     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1619     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1620     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1621         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1622         # and out of each queue plus one config interrupt and control vector queue
1623         my $vectors = $net->{queues} * 2 + 2;
1624         $tmpstr .= ",vectors=$vectors,mq=on";
1625     }
1626     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1627
1628     if (my $mtu = $net->{mtu}) {
1629         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1630             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1631             if ($mtu == 1) {
1632                  $mtu = $bridge_mtu;
1633             } elsif ($mtu < 576) {
1634                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1635             } elsif ($mtu > $bridge_mtu) {
1636                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1637             }
1638             $tmpstr .= ",host_mtu=$mtu";
1639         } else {
1640             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1641         }
1642     }
1643
1644     if ($use_old_bios_files) {
1645         my $romfile;
1646         if ($device eq 'virtio-net-pci') {
1647             $romfile = 'pxe-virtio.rom';
1648         } elsif ($device eq 'e1000') {
1649             $romfile = 'pxe-e1000.rom';
1650         } elsif ($device eq 'ne2k') {
1651             $romfile = 'pxe-ne2k_pci.rom';
1652         } elsif ($device eq 'pcnet') {
1653             $romfile = 'pxe-pcnet.rom';
1654         } elsif ($device eq 'rtl8139') {
1655             $romfile = 'pxe-rtl8139.rom';
1656         }
1657         $tmpstr .= ",romfile=$romfile" if $romfile;
1658     }
1659
1660     return $tmpstr;
1661 }
1662
1663 sub print_netdev_full {
1664     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1665
1666     my $i = '';
1667     if ($netid =~ m/^net(\d+)$/) {
1668         $i = int($1);
1669     }
1670
1671     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1672
1673     my $ifname = "tap${vmid}i$i";
1674
1675     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1676     die "interface name '$ifname' is too long (max 15 character)\n"
1677         if length($ifname) >= 16;
1678
1679     my $vhostparam = '';
1680     if (is_native($arch)) {
1681         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1682     }
1683
1684     my $vmname = $conf->{name} || "vm$vmid";
1685
1686     my $netdev = "";
1687     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1688
1689     if ($net->{bridge}) {
1690         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1691             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1692     } else {
1693         $netdev = "type=user,id=$netid,hostname=$vmname";
1694     }
1695
1696     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1697
1698     return $netdev;
1699 }
1700
1701 my $vga_map = {
1702     'cirrus' => 'cirrus-vga',
1703     'std' => 'VGA',
1704     'vmware' => 'vmware-svga',
1705     'virtio' => 'virtio-vga',
1706 };
1707
1708 sub print_vga_device {
1709     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1710
1711     my $type = $vga_map->{$vga->{type}};
1712     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1713         $type = 'virtio-gpu';
1714     }
1715     my $vgamem_mb = $vga->{memory};
1716
1717     my $max_outputs = '';
1718     if ($qxlnum) {
1719         $type = $id ? 'qxl' : 'qxl-vga';
1720
1721         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1722             # set max outputs so linux can have up to 4 qxl displays with one device
1723             if (min_version($machine_version, 4, 1)) {
1724                 $max_outputs = ",max_outputs=4";
1725             }
1726         }
1727     }
1728
1729     die "no devicetype for $vga->{type}\n" if !$type;
1730
1731     my $memory = "";
1732     if ($vgamem_mb) {
1733         if ($vga->{type} eq 'virtio') {
1734             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1735             $memory = ",max_hostmem=$bytes";
1736         } elsif ($qxlnum) {
1737             # from https://www.spice-space.org/multiple-monitors.html
1738             $memory = ",vgamem_mb=$vga->{memory}";
1739             my $ram = $vgamem_mb * 4;
1740             my $vram = $vgamem_mb * 2;
1741             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1742         } else {
1743             $memory = ",vgamem_mb=$vga->{memory}";
1744         }
1745     } elsif ($qxlnum && $id) {
1746         $memory = ",ram_size=67108864,vram_size=33554432";
1747     }
1748
1749     my $edidoff = "";
1750     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1751         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1752     }
1753
1754     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1755     my $vgaid = "vga" . ($id // '');
1756     my $pciaddr;
1757
1758     if ($q35 && $vgaid eq 'vga') {
1759         # the first display uses pcie.0 bus on q35 machines
1760         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1761     } else {
1762         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1763     }
1764
1765     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1766 }
1767
1768 sub parse_number_sets {
1769     my ($set) = @_;
1770     my $res = [];
1771     foreach my $part (split(/;/, $set)) {
1772         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1773             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1774             push @$res, [ $1, $2 ];
1775         } else {
1776             die "invalid range: $part\n";
1777         }
1778     }
1779     return $res;
1780 }
1781
1782 sub parse_numa {
1783     my ($data) = @_;
1784
1785     my $res = parse_property_string($numa_fmt, $data);
1786     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1787     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1788     return $res;
1789 }
1790
1791 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1792 sub parse_net {
1793     my ($data) = @_;
1794
1795     my $res = eval { parse_property_string($net_fmt, $data) };
1796     if ($@) {
1797         warn $@;
1798         return;
1799     }
1800     if (!defined($res->{macaddr})) {
1801         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1802         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1803     }
1804     return $res;
1805 }
1806
1807 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1808 sub parse_ipconfig {
1809     my ($data) = @_;
1810
1811     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1812     if ($@) {
1813         warn $@;
1814         return;
1815     }
1816
1817     if ($res->{gw} && !$res->{ip}) {
1818         warn 'gateway specified without specifying an IP address';
1819         return;
1820     }
1821     if ($res->{gw6} && !$res->{ip6}) {
1822         warn 'IPv6 gateway specified without specifying an IPv6 address';
1823         return;
1824     }
1825     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1826         warn 'gateway specified together with DHCP';
1827         return;
1828     }
1829     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1830         # gw6 + auto/dhcp
1831         warn "IPv6 gateway specified together with $res->{ip6} address";
1832         return;
1833     }
1834
1835     if (!$res->{ip} && !$res->{ip6}) {
1836         return { ip => 'dhcp', ip6 => 'dhcp' };
1837     }
1838
1839     return $res;
1840 }
1841
1842 sub print_net {
1843     my $net = shift;
1844
1845     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1846 }
1847
1848 sub add_random_macs {
1849     my ($settings) = @_;
1850
1851     foreach my $opt (keys %$settings) {
1852         next if $opt !~ m/^net(\d+)$/;
1853         my $net = parse_net($settings->{$opt});
1854         next if !$net;
1855         $settings->{$opt} = print_net($net);
1856     }
1857 }
1858
1859 sub vm_is_volid_owner {
1860     my ($storecfg, $vmid, $volid) = @_;
1861
1862     if ($volid !~  m|^/|) {
1863         my ($path, $owner);
1864         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1865         if ($owner && ($owner == $vmid)) {
1866             return 1;
1867         }
1868     }
1869
1870     return;
1871 }
1872
1873 sub vmconfig_register_unused_drive {
1874     my ($storecfg, $vmid, $conf, $drive) = @_;
1875
1876     if (drive_is_cloudinit($drive)) {
1877         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1878         warn $@ if $@;
1879     } elsif (!drive_is_cdrom($drive)) {
1880         my $volid = $drive->{file};
1881         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1882             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1883         }
1884     }
1885 }
1886
1887 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1888 my $smbios1_fmt = {
1889     uuid => {
1890         type => 'string',
1891         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1892         format_description => 'UUID',
1893         description => "Set SMBIOS1 UUID.",
1894         optional => 1,
1895     },
1896     version => {
1897         type => 'string',
1898         pattern => '[A-Za-z0-9+\/]+={0,2}',
1899         format_description => 'Base64 encoded string',
1900         description => "Set SMBIOS1 version.",
1901         optional => 1,
1902     },
1903     serial => {
1904         type => 'string',
1905         pattern => '[A-Za-z0-9+\/]+={0,2}',
1906         format_description => 'Base64 encoded string',
1907         description => "Set SMBIOS1 serial number.",
1908         optional => 1,
1909     },
1910     manufacturer => {
1911         type => 'string',
1912         pattern => '[A-Za-z0-9+\/]+={0,2}',
1913         format_description => 'Base64 encoded string',
1914         description => "Set SMBIOS1 manufacturer.",
1915         optional => 1,
1916     },
1917     product => {
1918         type => 'string',
1919         pattern => '[A-Za-z0-9+\/]+={0,2}',
1920         format_description => 'Base64 encoded string',
1921         description => "Set SMBIOS1 product ID.",
1922         optional => 1,
1923     },
1924     sku => {
1925         type => 'string',
1926         pattern => '[A-Za-z0-9+\/]+={0,2}',
1927         format_description => 'Base64 encoded string',
1928         description => "Set SMBIOS1 SKU string.",
1929         optional => 1,
1930     },
1931     family => {
1932         type => 'string',
1933         pattern => '[A-Za-z0-9+\/]+={0,2}',
1934         format_description => 'Base64 encoded string',
1935         description => "Set SMBIOS1 family string.",
1936         optional => 1,
1937     },
1938     base64 => {
1939         type => 'boolean',
1940         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1941         optional => 1,
1942     },
1943 };
1944
1945 sub parse_smbios1 {
1946     my ($data) = @_;
1947
1948     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1949     warn $@ if $@;
1950     return $res;
1951 }
1952
1953 sub print_smbios1 {
1954     my ($smbios1) = @_;
1955     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1956 }
1957
1958 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1959
1960 sub parse_watchdog {
1961     my ($value) = @_;
1962
1963     return if !$value;
1964
1965     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1966     warn $@ if $@;
1967     return $res;
1968 }
1969
1970 sub parse_guest_agent {
1971     my ($conf) = @_;
1972
1973     return {} if !defined($conf->{agent});
1974
1975     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
1976     warn $@ if $@;
1977
1978     # if the agent is disabled ignore the other potentially set properties
1979     return {} if !$res->{enabled};
1980     return $res;
1981 }
1982
1983 sub get_qga_key {
1984     my ($conf, $key) = @_;
1985     return undef if !defined($conf->{agent});
1986
1987     my $agent = parse_guest_agent($conf);
1988     return $agent->{$key};
1989 }
1990
1991 sub parse_vga {
1992     my ($value) = @_;
1993
1994     return {} if !$value;
1995     my $res = eval { parse_property_string($vga_fmt, $value) };
1996     warn $@ if $@;
1997     return $res;
1998 }
1999
2000 sub parse_rng {
2001     my ($value) = @_;
2002
2003     return if !$value;
2004
2005     my $res = eval { parse_property_string($rng_fmt, $value) };
2006     warn $@ if $@;
2007     return $res;
2008 }
2009
2010 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2011 sub verify_usb_device {
2012     my ($value, $noerr) = @_;
2013
2014     return $value if parse_usb_device($value);
2015
2016     return if $noerr;
2017
2018     die "unable to parse usb device\n";
2019 }
2020
2021 # add JSON properties for create and set function
2022 sub json_config_properties {
2023     my $prop = shift;
2024
2025     foreach my $opt (keys %$confdesc) {
2026         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2027             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2028         $prop->{$opt} = $confdesc->{$opt};
2029     }
2030
2031     return $prop;
2032 }
2033
2034 # return copy of $confdesc_cloudinit to generate documentation
2035 sub cloudinit_config_properties {
2036
2037     return dclone($confdesc_cloudinit);
2038 }
2039
2040 sub check_type {
2041     my ($key, $value) = @_;
2042
2043     die "unknown setting '$key'\n" if !$confdesc->{$key};
2044
2045     my $type = $confdesc->{$key}->{type};
2046
2047     if (!defined($value)) {
2048         die "got undefined value\n";
2049     }
2050
2051     if ($value =~ m/[\n\r]/) {
2052         die "property contains a line feed\n";
2053     }
2054
2055     if ($type eq 'boolean') {
2056         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2057         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2058         die "type check ('boolean') failed - got '$value'\n";
2059     } elsif ($type eq 'integer') {
2060         return int($1) if $value =~ m/^(\d+)$/;
2061         die "type check ('integer') failed - got '$value'\n";
2062     } elsif ($type eq 'number') {
2063         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2064         die "type check ('number') failed - got '$value'\n";
2065     } elsif ($type eq 'string') {
2066         if (my $fmt = $confdesc->{$key}->{format}) {
2067             PVE::JSONSchema::check_format($fmt, $value);
2068             return $value;
2069         }
2070         $value =~ s/^\"(.*)\"$/$1/;
2071         return $value;
2072     } else {
2073         die "internal error"
2074     }
2075 }
2076
2077 sub destroy_vm {
2078     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2079
2080     my $conf = PVE::QemuConfig->load_config($vmid);
2081
2082     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2083
2084     if ($conf->{template}) {
2085         # check if any base image is still used by a linked clone
2086         PVE::QemuConfig->foreach_volume($conf, sub {
2087                 my ($ds, $drive) = @_;
2088                 return if drive_is_cdrom($drive);
2089
2090                 my $volid = $drive->{file};
2091                 return if !$volid || $volid =~ m|^/|;
2092
2093                 die "base volume '$volid' is still in use by linked cloned\n"
2094                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2095
2096         });
2097     }
2098
2099     # only remove disks owned by this VM (referenced in the config)
2100     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2101         my ($ds, $drive) = @_;
2102         return if drive_is_cdrom($drive, 1);
2103
2104         my $volid = $drive->{file};
2105         return if !$volid || $volid =~ m|^/|;
2106
2107         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2108         return if !$path || !$owner || ($owner != $vmid);
2109
2110         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2111         warn "Could not remove disk '$volid', check manually: $@" if $@;
2112     });
2113
2114     if ($purge_unreferenced) { # also remove unreferenced disk
2115         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2116         PVE::Storage::foreach_volid($vmdisks, sub {
2117             my ($volid, $sid, $volname, $d) = @_;
2118             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2119             warn $@ if $@;
2120         });
2121     }
2122
2123     if (defined $replacement_conf) {
2124         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2125     } else {
2126         PVE::QemuConfig->destroy_config($vmid);
2127     }
2128 }
2129
2130 sub parse_vm_config {
2131     my ($filename, $raw) = @_;
2132
2133     return if !defined($raw);
2134
2135     my $res = {
2136         digest => Digest::SHA::sha1_hex($raw),
2137         snapshots => {},
2138         pending => {},
2139     };
2140
2141     $filename =~ m|/qemu-server/(\d+)\.conf$|
2142         || die "got strange filename '$filename'";
2143
2144     my $vmid = $1;
2145
2146     my $conf = $res;
2147     my $descr;
2148     my $section = '';
2149
2150     my @lines = split(/\n/, $raw);
2151     foreach my $line (@lines) {
2152         next if $line =~ m/^\s*$/;
2153
2154         if ($line =~ m/^\[PENDING\]\s*$/i) {
2155             $section = 'pending';
2156             if (defined($descr)) {
2157                 $descr =~ s/\s+$//;
2158                 $conf->{description} = $descr;
2159             }
2160             $descr = undef;
2161             $conf = $res->{$section} = {};
2162             next;
2163
2164         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2165             $section = $1;
2166             if (defined($descr)) {
2167                 $descr =~ s/\s+$//;
2168                 $conf->{description} = $descr;
2169             }
2170             $descr = undef;
2171             $conf = $res->{snapshots}->{$section} = {};
2172             next;
2173         }
2174
2175         if ($line =~ m/^\#(.*)\s*$/) {
2176             $descr = '' if !defined($descr);
2177             $descr .= PVE::Tools::decode_text($1) . "\n";
2178             next;
2179         }
2180
2181         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2182             $descr = '' if !defined($descr);
2183             $descr .= PVE::Tools::decode_text($2);
2184         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2185             $conf->{snapstate} = $1;
2186         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2187             my $key = $1;
2188             my $value = $2;
2189             $conf->{$key} = $value;
2190         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2191             my $value = $1;
2192             if ($section eq 'pending') {
2193                 $conf->{delete} = $value; # we parse this later
2194             } else {
2195                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2196             }
2197         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2198             my $key = $1;
2199             my $value = $2;
2200             eval { $value = check_type($key, $value); };
2201             if ($@) {
2202                 warn "vm $vmid - unable to parse value of '$key' - $@";
2203             } else {
2204                 $key = 'ide2' if $key eq 'cdrom';
2205                 my $fmt = $confdesc->{$key}->{format};
2206                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2207                     my $v = parse_drive($key, $value);
2208                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2209                         $v->{file} = $volid;
2210                         $value = print_drive($v);
2211                     } else {
2212                         warn "vm $vmid - unable to parse value of '$key'\n";
2213                         next;
2214                     }
2215                 }
2216
2217                 $conf->{$key} = $value;
2218             }
2219         } else {
2220             warn "vm $vmid - unable to parse config: $line\n";
2221         }
2222     }
2223
2224     if (defined($descr)) {
2225         $descr =~ s/\s+$//;
2226         $conf->{description} = $descr;
2227     }
2228     delete $res->{snapstate}; # just to be sure
2229
2230     return $res;
2231 }
2232
2233 sub write_vm_config {
2234     my ($filename, $conf) = @_;
2235
2236     delete $conf->{snapstate}; # just to be sure
2237
2238     if ($conf->{cdrom}) {
2239         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2240         $conf->{ide2} = $conf->{cdrom};
2241         delete $conf->{cdrom};
2242     }
2243
2244     # we do not use 'smp' any longer
2245     if ($conf->{sockets}) {
2246         delete $conf->{smp};
2247     } elsif ($conf->{smp}) {
2248         $conf->{sockets} = $conf->{smp};
2249         delete $conf->{cores};
2250         delete $conf->{smp};
2251     }
2252
2253     my $used_volids = {};
2254
2255     my $cleanup_config = sub {
2256         my ($cref, $pending, $snapname) = @_;
2257
2258         foreach my $key (keys %$cref) {
2259             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2260                 $key eq 'snapstate' || $key eq 'pending';
2261             my $value = $cref->{$key};
2262             if ($key eq 'delete') {
2263                 die "propertry 'delete' is only allowed in [PENDING]\n"
2264                     if !$pending;
2265                 # fixme: check syntax?
2266                 next;
2267             }
2268             eval { $value = check_type($key, $value); };
2269             die "unable to parse value of '$key' - $@" if $@;
2270
2271             $cref->{$key} = $value;
2272
2273             if (!$snapname && is_valid_drivename($key)) {
2274                 my $drive = parse_drive($key, $value);
2275                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2276             }
2277         }
2278     };
2279
2280     &$cleanup_config($conf);
2281
2282     &$cleanup_config($conf->{pending}, 1);
2283
2284     foreach my $snapname (keys %{$conf->{snapshots}}) {
2285         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2286         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2287     }
2288
2289     # remove 'unusedX' settings if we re-add a volume
2290     foreach my $key (keys %$conf) {
2291         my $value = $conf->{$key};
2292         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2293             delete $conf->{$key};
2294         }
2295     }
2296
2297     my $generate_raw_config = sub {
2298         my ($conf, $pending) = @_;
2299
2300         my $raw = '';
2301
2302         # add description as comment to top of file
2303         if (defined(my $descr = $conf->{description})) {
2304             if ($descr) {
2305                 foreach my $cl (split(/\n/, $descr)) {
2306                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2307                 }
2308             } else {
2309                 $raw .= "#\n" if $pending;
2310             }
2311         }
2312
2313         foreach my $key (sort keys %$conf) {
2314             next if $key =~ /^(digest|description|pending|snapshots)$/;
2315             $raw .= "$key: $conf->{$key}\n";
2316         }
2317         return $raw;
2318     };
2319
2320     my $raw = &$generate_raw_config($conf);
2321
2322     if (scalar(keys %{$conf->{pending}})){
2323         $raw .= "\n[PENDING]\n";
2324         $raw .= &$generate_raw_config($conf->{pending}, 1);
2325     }
2326
2327     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2328         $raw .= "\n[$snapname]\n";
2329         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2330     }
2331
2332     return $raw;
2333 }
2334
2335 sub load_defaults {
2336
2337     my $res = {};
2338
2339     # we use static defaults from our JSON schema configuration
2340     foreach my $key (keys %$confdesc) {
2341         if (defined(my $default = $confdesc->{$key}->{default})) {
2342             $res->{$key} = $default;
2343         }
2344     }
2345
2346     return $res;
2347 }
2348
2349 sub config_list {
2350     my $vmlist = PVE::Cluster::get_vmlist();
2351     my $res = {};
2352     return $res if !$vmlist || !$vmlist->{ids};
2353     my $ids = $vmlist->{ids};
2354     my $nodename = nodename();
2355
2356     foreach my $vmid (keys %$ids) {
2357         my $d = $ids->{$vmid};
2358         next if !$d->{node} || $d->{node} ne $nodename;
2359         next if !$d->{type} || $d->{type} ne 'qemu';
2360         $res->{$vmid}->{exists} = 1;
2361     }
2362     return $res;
2363 }
2364
2365 # test if VM uses local resources (to prevent migration)
2366 sub check_local_resources {
2367     my ($conf, $noerr) = @_;
2368
2369     my @loc_res = ();
2370
2371     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2372     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2373
2374     push @loc_res, "ivshmem" if $conf->{ivshmem};
2375
2376     foreach my $k (keys %$conf) {
2377         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2378         # sockets are safe: they will recreated be on the target side post-migrate
2379         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2380         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2381     }
2382
2383     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2384
2385     return \@loc_res;
2386 }
2387
2388 # check if used storages are available on all nodes (use by migrate)
2389 sub check_storage_availability {
2390     my ($storecfg, $conf, $node) = @_;
2391
2392     PVE::QemuConfig->foreach_volume($conf, sub {
2393         my ($ds, $drive) = @_;
2394
2395         my $volid = $drive->{file};
2396         return if !$volid;
2397
2398         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2399         return if !$sid;
2400
2401         # check if storage is available on both nodes
2402         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2403         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2404    });
2405 }
2406
2407 # list nodes where all VM images are available (used by has_feature API)
2408 sub shared_nodes {
2409     my ($conf, $storecfg) = @_;
2410
2411     my $nodelist = PVE::Cluster::get_nodelist();
2412     my $nodehash = { map { $_ => 1 } @$nodelist };
2413     my $nodename = nodename();
2414
2415     PVE::QemuConfig->foreach_volume($conf, sub {
2416         my ($ds, $drive) = @_;
2417
2418         my $volid = $drive->{file};
2419         return if !$volid;
2420
2421         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2422         if ($storeid) {
2423             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2424             if ($scfg->{disable}) {
2425                 $nodehash = {};
2426             } elsif (my $avail = $scfg->{nodes}) {
2427                 foreach my $node (keys %$nodehash) {
2428                     delete $nodehash->{$node} if !$avail->{$node};
2429                 }
2430             } elsif (!$scfg->{shared}) {
2431                 foreach my $node (keys %$nodehash) {
2432                     delete $nodehash->{$node} if $node ne $nodename
2433                 }
2434             }
2435         }
2436     });
2437
2438     return $nodehash
2439 }
2440
2441 sub check_local_storage_availability {
2442     my ($conf, $storecfg) = @_;
2443
2444     my $nodelist = PVE::Cluster::get_nodelist();
2445     my $nodehash = { map { $_ => {} } @$nodelist };
2446
2447     PVE::QemuConfig->foreach_volume($conf, sub {
2448         my ($ds, $drive) = @_;
2449
2450         my $volid = $drive->{file};
2451         return if !$volid;
2452
2453         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2454         if ($storeid) {
2455             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2456
2457             if ($scfg->{disable}) {
2458                 foreach my $node (keys %$nodehash) {
2459                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2460                 }
2461             } elsif (my $avail = $scfg->{nodes}) {
2462                 foreach my $node (keys %$nodehash) {
2463                     if (!$avail->{$node}) {
2464                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2465                     }
2466                 }
2467             }
2468         }
2469     });
2470
2471     foreach my $node (values %$nodehash) {
2472         if (my $unavail = $node->{unavailable_storages}) {
2473             $node->{unavailable_storages} = [ sort keys %$unavail ];
2474         }
2475     }
2476
2477     return $nodehash
2478 }
2479
2480 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2481 sub check_running {
2482     my ($vmid, $nocheck, $node) = @_;
2483
2484     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2485     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2486 }
2487
2488 sub vzlist {
2489
2490     my $vzlist = config_list();
2491
2492     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2493
2494     while (defined(my $de = $fd->read)) {
2495         next if $de !~ m/^(\d+)\.pid$/;
2496         my $vmid = $1;
2497         next if !defined($vzlist->{$vmid});
2498         if (my $pid = check_running($vmid)) {
2499             $vzlist->{$vmid}->{pid} = $pid;
2500         }
2501     }
2502
2503     return $vzlist;
2504 }
2505
2506 our $vmstatus_return_properties = {
2507     vmid => get_standard_option('pve-vmid'),
2508     status => {
2509         description => "Qemu process status.",
2510         type => 'string',
2511         enum => ['stopped', 'running'],
2512     },
2513     maxmem => {
2514         description => "Maximum memory in bytes.",
2515         type => 'integer',
2516         optional => 1,
2517         renderer => 'bytes',
2518     },
2519     maxdisk => {
2520         description => "Root disk size in bytes.",
2521         type => 'integer',
2522         optional => 1,
2523         renderer => 'bytes',
2524     },
2525     name => {
2526         description => "VM name.",
2527         type => 'string',
2528         optional => 1,
2529     },
2530     qmpstatus => {
2531         description => "Qemu QMP agent status.",
2532         type => 'string',
2533         optional => 1,
2534     },
2535     pid => {
2536         description => "PID of running qemu process.",
2537         type => 'integer',
2538         optional => 1,
2539     },
2540     uptime => {
2541         description => "Uptime.",
2542         type => 'integer',
2543         optional => 1,
2544         renderer => 'duration',
2545     },
2546     cpus => {
2547         description => "Maximum usable CPUs.",
2548         type => 'number',
2549         optional => 1,
2550     },
2551     lock => {
2552         description => "The current config lock, if any.",
2553         type => 'string',
2554         optional => 1,
2555     },
2556     tags => {
2557         description  => "The current configured tags, if any",
2558         type => 'string',
2559         optional => 1,
2560     },
2561     'running-machine' => {
2562         description => "The currently running machine type (if running).",
2563         type => 'string',
2564         optional => 1,
2565     },
2566     'running-qemu' => {
2567         description => "The currently running QEMU version (if running).",
2568         type => 'string',
2569         optional => 1,
2570     },
2571 };
2572
2573 my $last_proc_pid_stat;
2574
2575 # get VM status information
2576 # This must be fast and should not block ($full == false)
2577 # We only query KVM using QMP if $full == true (this can be slow)
2578 sub vmstatus {
2579     my ($opt_vmid, $full) = @_;
2580
2581     my $res = {};
2582
2583     my $storecfg = PVE::Storage::config();
2584
2585     my $list = vzlist();
2586     my $defaults = load_defaults();
2587
2588     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2589
2590     my $cpucount = $cpuinfo->{cpus} || 1;
2591
2592     foreach my $vmid (keys %$list) {
2593         next if $opt_vmid && ($vmid ne $opt_vmid);
2594
2595         my $conf = PVE::QemuConfig->load_config($vmid);
2596
2597         my $d = { vmid => $vmid };
2598         $d->{pid} = $list->{$vmid}->{pid};
2599
2600         # fixme: better status?
2601         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2602
2603         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2604         if (defined($size)) {
2605             $d->{disk} = 0; # no info available
2606             $d->{maxdisk} = $size;
2607         } else {
2608             $d->{disk} = 0;
2609             $d->{maxdisk} = 0;
2610         }
2611
2612         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2613             * ($conf->{cores} || $defaults->{cores});
2614         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2615         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2616
2617         $d->{name} = $conf->{name} || "VM $vmid";
2618         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2619             : $defaults->{memory}*(1024*1024);
2620
2621         if ($conf->{balloon}) {
2622             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2623             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2624                 : $defaults->{shares};
2625         }
2626
2627         $d->{uptime} = 0;
2628         $d->{cpu} = 0;
2629         $d->{mem} = 0;
2630
2631         $d->{netout} = 0;
2632         $d->{netin} = 0;
2633
2634         $d->{diskread} = 0;
2635         $d->{diskwrite} = 0;
2636
2637         $d->{template} = PVE::QemuConfig->is_template($conf);
2638
2639         $d->{serial} = 1 if conf_has_serial($conf);
2640         $d->{lock} = $conf->{lock} if $conf->{lock};
2641         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2642
2643         $res->{$vmid} = $d;
2644     }
2645
2646     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2647     foreach my $dev (keys %$netdev) {
2648         next if $dev !~ m/^tap([1-9]\d*)i/;
2649         my $vmid = $1;
2650         my $d = $res->{$vmid};
2651         next if !$d;
2652
2653         $d->{netout} += $netdev->{$dev}->{receive};
2654         $d->{netin} += $netdev->{$dev}->{transmit};
2655
2656         if ($full) {
2657             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2658             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2659         }
2660
2661     }
2662
2663     my $ctime = gettimeofday;
2664
2665     foreach my $vmid (keys %$list) {
2666
2667         my $d = $res->{$vmid};
2668         my $pid = $d->{pid};
2669         next if !$pid;
2670
2671         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2672         next if !$pstat; # not running
2673
2674         my $used = $pstat->{utime} + $pstat->{stime};
2675
2676         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2677
2678         if ($pstat->{vsize}) {
2679             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2680         }
2681
2682         my $old = $last_proc_pid_stat->{$pid};
2683         if (!$old) {
2684             $last_proc_pid_stat->{$pid} = {
2685                 time => $ctime,
2686                 used => $used,
2687                 cpu => 0,
2688             };
2689             next;
2690         }
2691
2692         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2693
2694         if ($dtime > 1000) {
2695             my $dutime = $used -  $old->{used};
2696
2697             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2698             $last_proc_pid_stat->{$pid} = {
2699                 time => $ctime,
2700                 used => $used,
2701                 cpu => $d->{cpu},
2702             };
2703         } else {
2704             $d->{cpu} = $old->{cpu};
2705         }
2706     }
2707
2708     return $res if !$full;
2709
2710     my $qmpclient = PVE::QMPClient->new();
2711
2712     my $ballooncb = sub {
2713         my ($vmid, $resp) = @_;
2714
2715         my $info = $resp->{'return'};
2716         return if !$info->{max_mem};
2717
2718         my $d = $res->{$vmid};
2719
2720         # use memory assigned to VM
2721         $d->{maxmem} = $info->{max_mem};
2722         $d->{balloon} = $info->{actual};
2723
2724         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2725             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2726             $d->{freemem} = $info->{free_mem};
2727         }
2728
2729         $d->{ballooninfo} = $info;
2730     };
2731
2732     my $blockstatscb = sub {
2733         my ($vmid, $resp) = @_;
2734         my $data = $resp->{'return'} || [];
2735         my $totalrdbytes = 0;
2736         my $totalwrbytes = 0;
2737
2738         for my $blockstat (@$data) {
2739             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2740             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2741
2742             $blockstat->{device} =~ s/drive-//;
2743             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2744         }
2745         $res->{$vmid}->{diskread} = $totalrdbytes;
2746         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2747     };
2748
2749     my $machinecb = sub {
2750         my ($vmid, $resp) = @_;
2751         my $data = $resp->{'return'} || [];
2752
2753         $res->{$vmid}->{'running-machine'} =
2754             PVE::QemuServer::Machine::current_from_query_machines($data);
2755     };
2756
2757     my $versioncb = sub {
2758         my ($vmid, $resp) = @_;
2759         my $data = $resp->{'return'} // {};
2760         my $version = 'unknown';
2761
2762         if (my $v = $data->{qemu}) {
2763             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2764         }
2765
2766         $res->{$vmid}->{'running-qemu'} = $version;
2767     };
2768
2769     my $statuscb = sub {
2770         my ($vmid, $resp) = @_;
2771
2772         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2773         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2774         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2775         # this fails if ballon driver is not loaded, so this must be
2776         # the last commnand (following command are aborted if this fails).
2777         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2778
2779         my $status = 'unknown';
2780         if (!defined($status = $resp->{'return'}->{status})) {
2781             warn "unable to get VM status\n";
2782             return;
2783         }
2784
2785         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2786     };
2787
2788     foreach my $vmid (keys %$list) {
2789         next if $opt_vmid && ($vmid ne $opt_vmid);
2790         next if !$res->{$vmid}->{pid}; # not running
2791         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2792     }
2793
2794     $qmpclient->queue_execute(undef, 2);
2795
2796     foreach my $vmid (keys %$list) {
2797         next if $opt_vmid && ($vmid ne $opt_vmid);
2798         next if !$res->{$vmid}->{pid}; #not running
2799
2800         # we can't use the $qmpclient since it might have already aborted on
2801         # 'query-balloon', but this might also fail for older versions...
2802         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2803         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2804     }
2805
2806     foreach my $vmid (keys %$list) {
2807         next if $opt_vmid && ($vmid ne $opt_vmid);
2808         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2809     }
2810
2811     return $res;
2812 }
2813
2814 sub conf_has_serial {
2815     my ($conf) = @_;
2816
2817     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2818         if ($conf->{"serial$i"}) {
2819             return 1;
2820         }
2821     }
2822
2823     return 0;
2824 }
2825
2826 sub conf_has_audio {
2827     my ($conf, $id) = @_;
2828
2829     $id //= 0;
2830     my $audio = $conf->{"audio$id"};
2831     return if !defined($audio);
2832
2833     my $audioproperties = parse_property_string($audio_fmt, $audio);
2834     my $audiodriver = $audioproperties->{driver} // 'spice';
2835
2836     return {
2837         dev => $audioproperties->{device},
2838         dev_id => "audiodev$id",
2839         backend => $audiodriver,
2840         backend_id => "$audiodriver-backend${id}",
2841     };
2842 }
2843
2844 sub audio_devs {
2845     my ($audio, $audiopciaddr, $machine_version) = @_;
2846
2847     my $devs = [];
2848
2849     my $id = $audio->{dev_id};
2850     my $audiodev = "";
2851     if (min_version($machine_version, 4, 2)) {
2852         $audiodev = ",audiodev=$audio->{backend_id}";
2853     }
2854
2855     if ($audio->{dev} eq 'AC97') {
2856         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2857     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2858         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2859         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2860         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2861     } else {
2862         die "unkown audio device '$audio->{dev}', implement me!";
2863     }
2864
2865     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2866
2867     return $devs;
2868 }
2869
2870 sub vga_conf_has_spice {
2871     my ($vga) = @_;
2872
2873     my $vgaconf = parse_vga($vga);
2874     my $vgatype = $vgaconf->{type};
2875     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2876
2877     return $1 || 1;
2878 }
2879
2880 sub is_native($) {
2881     my ($arch) = @_;
2882     return get_host_arch() eq $arch;
2883 }
2884
2885 sub get_vm_arch {
2886     my ($conf) = @_;
2887     return $conf->{arch} // get_host_arch();
2888 }
2889
2890 my $default_machines = {
2891     x86_64 => 'pc',
2892     aarch64 => 'virt',
2893 };
2894
2895 sub get_installed_machine_version {
2896     my ($kvmversion) = @_;
2897     $kvmversion = kvm_user_version() if !defined($kvmversion);
2898     $kvmversion =~ m/^(\d+\.\d+)/;
2899     return $1;
2900 }
2901
2902 sub windows_get_pinned_machine_version {
2903     my ($machine, $base_version, $kvmversion) = @_;
2904
2905     my $pin_version = $base_version;
2906     if (!defined($base_version) ||
2907         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2908     ) {
2909         $pin_version = get_installed_machine_version($kvmversion);
2910     }
2911     if (!$machine || $machine eq 'pc') {
2912         $machine = "pc-i440fx-$pin_version";
2913     } elsif ($machine eq 'q35') {
2914         $machine = "pc-q35-$pin_version";
2915     } elsif ($machine eq 'virt') {
2916         $machine = "virt-$pin_version";
2917     } else {
2918         warn "unknown machine type '$machine', not touching that!\n";
2919     }
2920
2921     return $machine;
2922 }
2923
2924 sub get_vm_machine {
2925     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2926
2927     my $machine = $forcemachine || $conf->{machine};
2928
2929     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2930         $kvmversion //= kvm_user_version();
2931         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2932         # layout which confuses windows quite a bit and may result in various regressions..
2933         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2934         if (windows_version($conf->{ostype})) {
2935             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2936         }
2937         $arch //= 'x86_64';
2938         $machine ||= $default_machines->{$arch};
2939         if ($add_pve_version) {
2940             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2941             $machine .= "+pve$pvever";
2942         }
2943     }
2944
2945     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2946         # for version-pinned machines that do not include a pve-version (e.g.
2947         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2948         $machine .= '+pve0';
2949     }
2950
2951     return $machine;
2952 }
2953
2954 sub get_ovmf_files($) {
2955     my ($arch) = @_;
2956
2957     my $ovmf = $OVMF->{$arch}
2958         or die "no OVMF images known for architecture '$arch'\n";
2959
2960     return @$ovmf;
2961 }
2962
2963 my $Arch2Qemu = {
2964     aarch64 => '/usr/bin/qemu-system-aarch64',
2965     x86_64 => '/usr/bin/qemu-system-x86_64',
2966 };
2967 sub get_command_for_arch($) {
2968     my ($arch) = @_;
2969     return '/usr/bin/kvm' if is_native($arch);
2970
2971     my $cmd = $Arch2Qemu->{$arch}
2972         or die "don't know how to emulate architecture '$arch'\n";
2973     return $cmd;
2974 }
2975
2976 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2977 # to use in a QEMU command line (-cpu element), first array_intersect the result
2978 # of query_supported_ with query_understood_. This is necessary because:
2979 #
2980 # a) query_understood_ returns flags the host cannot use and
2981 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2982 #    flags, but CPU settings - with most of them being flags. Those settings
2983 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2984 #
2985 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2986 # expensive. If you need the value returned from this, you can get it much
2987 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2988 # $accel being 'kvm' or 'tcg'.
2989 #
2990 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2991 # changes, automatically populating pmxcfs.
2992 #
2993 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2994 # since kvm and tcg machines support different flags
2995 #
2996 sub query_supported_cpu_flags {
2997     my ($arch) = @_;
2998
2999     $arch //= get_host_arch();
3000     my $default_machine = $default_machines->{$arch};
3001
3002     my $flags = {};
3003
3004     # FIXME: Once this is merged, the code below should work for ARM as well:
3005     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3006     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3007         $arch eq "aarch64";
3008
3009     my $kvm_supported = defined(kvm_version());
3010     my $qemu_cmd = get_command_for_arch($arch);
3011     my $fakevmid = -1;
3012     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3013
3014     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3015     my $query_supported_run_qemu = sub {
3016         my ($kvm) = @_;
3017
3018         my $flags = {};
3019         my $cmd = [
3020             $qemu_cmd,
3021             '-machine', $default_machine,
3022             '-display', 'none',
3023             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3024             '-mon', 'chardev=qmp,mode=control',
3025             '-pidfile', $pidfile,
3026             '-S', '-daemonize'
3027         ];
3028
3029         if (!$kvm) {
3030             push @$cmd, '-accel', 'tcg';
3031         }
3032
3033         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3034         die "QEMU flag querying VM exited with code " . $rc if $rc;
3035
3036         eval {
3037             my $cmd_result = mon_cmd(
3038                 $fakevmid,
3039                 'query-cpu-model-expansion',
3040                 type => 'full',
3041                 model => { name => 'host' }
3042             );
3043
3044             my $props = $cmd_result->{model}->{props};
3045             foreach my $prop (keys %$props) {
3046                 next if $props->{$prop} ne '1';
3047                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3048                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3049                 # We only keep those with underscores, to match /proc/cpuinfo
3050                 $prop =~ s/\.|-/_/g;
3051                 $flags->{$prop} = 1;
3052             }
3053         };
3054         my $err = $@;
3055
3056         # force stop with 10 sec timeout and 'nocheck'
3057         # always stop, even if QMP failed
3058         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3059
3060         die $err if $err;
3061
3062         return [ sort keys %$flags ];
3063     };
3064
3065     # We need to query QEMU twice, since KVM and TCG have different supported flags
3066     PVE::QemuConfig->lock_config($fakevmid, sub {
3067         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3068         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3069
3070         if ($kvm_supported) {
3071             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3072             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3073         }
3074     });
3075
3076     return $flags;
3077 }
3078
3079 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3080 my $understood_cpu_flag_dir = "/usr/share/kvm";
3081 sub query_understood_cpu_flags {
3082     my $arch = get_host_arch();
3083     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3084
3085     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3086         if ! -e $filepath;
3087
3088     my $raw = file_get_contents($filepath);
3089     $raw =~ s/^\s+|\s+$//g;
3090     my @flags = split(/\s+/, $raw);
3091
3092     return \@flags;
3093 }
3094
3095 sub config_to_command {
3096     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3097
3098     my $cmd = [];
3099     my $globalFlags = [];
3100     my $machineFlags = [];
3101     my $rtcFlags = [];
3102     my $devices = [];
3103     my $pciaddr = '';
3104     my $bridges = {};
3105     my $ostype = $conf->{ostype};
3106     my $winversion = windows_version($ostype);
3107     my $kvm = $conf->{kvm};
3108     my $nodename = nodename();
3109
3110     my $arch = get_vm_arch($conf);
3111     my $kvm_binary = get_command_for_arch($arch);
3112     my $kvmver = kvm_user_version($kvm_binary);
3113
3114     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3115         $kvmver //= "undefined";
3116         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3117     }
3118
3119     my $add_pve_version = min_version($kvmver, 4, 1);
3120
3121     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3122     my $machine_version = extract_version($machine_type, $kvmver);
3123     $kvm //= 1 if is_native($arch);
3124
3125     $machine_version =~ m/(\d+)\.(\d+)/;
3126     my ($machine_major, $machine_minor) = ($1, $2);
3127
3128     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3129         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3130     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3131         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3132             ." please upgrade node '$nodename'\n"
3133     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3134         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3135         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3136             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3137             ." node '$nodename'\n";
3138     }
3139
3140     # if a specific +pve version is required for a feature, use $version_guard
3141     # instead of min_version to allow machines to be run with the minimum
3142     # required version
3143     my $required_pve_version = 0;
3144     my $version_guard = sub {
3145         my ($major, $minor, $pve) = @_;
3146         return 0 if !min_version($machine_version, $major, $minor, $pve);
3147         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3148         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3149         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3150         return 1;
3151     };
3152
3153     if ($kvm && !defined kvm_version()) {
3154         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3155             ." or enable in BIOS.\n";
3156     }
3157
3158     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3159     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3160     my $use_old_bios_files = undef;
3161     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3162
3163     my $cpuunits = defined($conf->{cpuunits}) ?
3164             $conf->{cpuunits} : $defaults->{cpuunits};
3165
3166     push @$cmd, $kvm_binary;
3167
3168     push @$cmd, '-id', $vmid;
3169
3170     my $vmname = $conf->{name} || "vm$vmid";
3171
3172     push @$cmd, '-name', $vmname;
3173
3174     push @$cmd, '-no-shutdown';
3175
3176     my $use_virtio = 0;
3177
3178     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3179     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3180     push @$cmd, '-mon', "chardev=qmp,mode=control";
3181
3182     if (min_version($machine_version, 2, 12)) {
3183         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3184         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3185     }
3186
3187     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3188
3189     push @$cmd, '-daemonize';
3190
3191     if ($conf->{smbios1}) {
3192         my $smbios_conf = parse_smbios1($conf->{smbios1});
3193         if ($smbios_conf->{base64}) {
3194             # Do not pass base64 flag to qemu
3195             delete $smbios_conf->{base64};
3196             my $smbios_string = "";
3197             foreach my $key (keys %$smbios_conf) {
3198                 my $value;
3199                 if ($key eq "uuid") {
3200                     $value = $smbios_conf->{uuid}
3201                 } else {
3202                     $value = decode_base64($smbios_conf->{$key});
3203                 }
3204                 # qemu accepts any binary data, only commas need escaping by double comma
3205                 $value =~ s/,/,,/g;
3206                 $smbios_string .= "," . $key . "=" . $value if $value;
3207             }
3208             push @$cmd, '-smbios', "type=1" . $smbios_string;
3209         } else {
3210             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3211         }
3212     }
3213
3214     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3215         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3216         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3217
3218         my ($path, $format);
3219         if (my $efidisk = $conf->{efidisk0}) {
3220             my $d = parse_drive('efidisk0', $efidisk);
3221             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3222             $format = $d->{format};
3223             if ($storeid) {
3224                 $path = PVE::Storage::path($storecfg, $d->{file});
3225                 if (!defined($format)) {
3226                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3227                     $format = qemu_img_format($scfg, $volname);
3228                 }
3229             } else {
3230                 $path = $d->{file};
3231                 die "efidisk format must be specified\n"
3232                     if !defined($format);
3233             }
3234         } else {
3235             warn "no efidisk configured! Using temporary efivars disk.\n";
3236             $path = "/tmp/$vmid-ovmf.fd";
3237             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3238             $format = 'raw';
3239         }
3240
3241         my $size_str = "";
3242
3243         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3244             $size_str = ",size=" . (-s $ovmf_vars);
3245         }
3246
3247         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3248         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3249     }
3250
3251     # load q35 config
3252     if ($q35) {
3253         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3254         if (min_version($machine_version, 4, 0)) {
3255             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3256         } else {
3257             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3258         }
3259     }
3260
3261     if ($conf->{vmgenid}) {
3262         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3263     }
3264
3265     # add usb controllers
3266     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3267         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3268     push @$devices, @usbcontrollers if @usbcontrollers;
3269     my $vga = parse_vga($conf->{vga});
3270
3271     my $qxlnum = vga_conf_has_spice($conf->{vga});
3272     $vga->{type} = 'qxl' if $qxlnum;
3273
3274     if (!$vga->{type}) {
3275         if ($arch eq 'aarch64') {
3276             $vga->{type} = 'virtio';
3277         } elsif (min_version($machine_version, 2, 9)) {
3278             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3279         } else {
3280             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3281         }
3282     }
3283
3284     # enable absolute mouse coordinates (needed by vnc)
3285     my $tablet;
3286     if (defined($conf->{tablet})) {
3287         $tablet = $conf->{tablet};
3288     } else {
3289         $tablet = $defaults->{tablet};
3290         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3291         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3292     }
3293
3294     if ($tablet) {
3295         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3296         my $kbd = print_keyboarddevice_full($conf, $arch);
3297         push @$devices, '-device', $kbd if defined($kbd);
3298     }
3299
3300     my $bootorder = device_bootorder($conf);
3301
3302     # host pci device passthrough
3303     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3304         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3305
3306     # usb devices
3307     my $usb_dev_features = {};
3308     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3309
3310     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3311         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3312     push @$devices, @usbdevices if @usbdevices;
3313
3314     # serial devices
3315     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3316         if (my $path = $conf->{"serial$i"}) {
3317             if ($path eq 'socket') {
3318                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3319                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3320                 # On aarch64, serial0 is the UART device. Qemu only allows
3321                 # connecting UART devices via the '-serial' command line, as
3322                 # the device has a fixed slot on the hardware...
3323                 if ($arch eq 'aarch64' && $i == 0) {
3324                     push @$devices, '-serial', "chardev:serial$i";
3325                 } else {
3326                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3327                 }
3328             } else {
3329                 die "no such serial device\n" if ! -c $path;
3330                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3331                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3332             }
3333         }
3334     }
3335
3336     # parallel devices
3337     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3338         if (my $path = $conf->{"parallel$i"}) {
3339             die "no such parallel device\n" if ! -c $path;
3340             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3341             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3342             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3343         }
3344     }
3345
3346     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3347         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3348         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3349         push @$devices, @$audio_devs;
3350     }
3351
3352     my $sockets = 1;
3353     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3354     $sockets = $conf->{sockets} if  $conf->{sockets};
3355
3356     my $cores = $conf->{cores} || 1;
3357
3358     my $maxcpus = $sockets * $cores;
3359
3360     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3361
3362     my $allowed_vcpus = $cpuinfo->{cpus};
3363
3364     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3365         if ($allowed_vcpus < $maxcpus);
3366
3367     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3368
3369         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3370         for (my $i = 2; $i <= $vcpus; $i++)  {
3371             my $cpustr = print_cpu_device($conf,$i);
3372             push @$cmd, '-device', $cpustr;
3373         }
3374
3375     } else {
3376
3377         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3378     }
3379     push @$cmd, '-nodefaults';
3380
3381     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3382
3383     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3384
3385     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3386
3387     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3388         push @$devices, '-device', print_vga_device(
3389             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3390         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3391         push @$cmd,  '-vnc', "unix:$socket,password";
3392     } else {
3393         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3394         push @$cmd, '-nographic';
3395     }
3396
3397     # time drift fix
3398     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3399     my $useLocaltime = $conf->{localtime};
3400
3401     if ($winversion >= 5) { # windows
3402         $useLocaltime = 1 if !defined($conf->{localtime});
3403
3404         # use time drift fix when acpi is enabled
3405         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3406             $tdf = 1 if !defined($conf->{tdf});
3407         }
3408     }
3409
3410     if ($winversion >= 6) {
3411         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3412         push @$cmd, '-no-hpet';
3413     }
3414
3415     push @$rtcFlags, 'driftfix=slew' if $tdf;
3416
3417     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3418         push @$rtcFlags, "base=$conf->{startdate}";
3419     } elsif ($useLocaltime) {
3420         push @$rtcFlags, 'base=localtime';
3421     }
3422
3423     if ($forcecpu) {
3424         push @$cmd, '-cpu', $forcecpu;
3425     } else {
3426         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3427     }
3428
3429     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3430
3431     push @$cmd, '-S' if $conf->{freeze};
3432
3433     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3434
3435     my $guest_agent = parse_guest_agent($conf);
3436
3437     if ($guest_agent->{enabled}) {
3438         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3439         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3440
3441         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3442             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3443             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3444             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3445         } elsif ($guest_agent->{type} eq 'isa') {
3446             push @$devices, '-device', "isa-serial,chardev=qga0";
3447         }
3448     }
3449
3450     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3451     if ($rng && $version_guard->(4, 1, 2)) {
3452         check_rng_source($rng->{source});
3453
3454         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3455         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3456         my $limiter_str = "";
3457         if ($max_bytes) {
3458             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3459         }
3460
3461         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3462         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3463         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3464     }
3465
3466     my $spice_port;
3467
3468     if ($qxlnum) {
3469         if ($qxlnum > 1) {
3470             if ($winversion){
3471                 for (my $i = 1; $i < $qxlnum; $i++){
3472                     push @$devices, '-device', print_vga_device(
3473                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3474                 }
3475             } else {
3476                 # assume other OS works like Linux
3477                 my ($ram, $vram) = ("134217728", "67108864");
3478                 if ($vga->{memory}) {
3479                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3480                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3481                 }
3482                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3483                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3484             }
3485         }
3486
3487         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3488
3489         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3490         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3491         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3492
3493         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3494         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3495         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3496
3497         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3498         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3499
3500         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3501         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3502         if ($spice_enhancement->{foldersharing}) {
3503             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3504             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3505         }
3506
3507         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3508         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3509             if $spice_enhancement->{videostreaming};
3510
3511         push @$devices, '-spice', "$spice_opts";
3512     }
3513
3514     # enable balloon by default, unless explicitly disabled
3515     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3516         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3517         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3518     }
3519
3520     if ($conf->{watchdog}) {
3521         my $wdopts = parse_watchdog($conf->{watchdog});
3522         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3523         my $watchdog = $wdopts->{model} || 'i6300esb';
3524         push @$devices, '-device', "$watchdog$pciaddr";
3525         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3526     }
3527
3528     my $vollist = [];
3529     my $scsicontroller = {};
3530     my $ahcicontroller = {};
3531     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3532
3533     # Add iscsi initiator name if available
3534     if (my $initiator = get_initiator_name()) {
3535         push @$devices, '-iscsi', "initiator-name=$initiator";
3536     }
3537
3538     PVE::QemuConfig->foreach_volume($conf, sub {
3539         my ($ds, $drive) = @_;
3540
3541         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3542             push @$vollist, $drive->{file};
3543         }
3544
3545         # ignore efidisk here, already added in bios/fw handling code above
3546         return if $drive->{interface} eq 'efidisk';
3547
3548         $use_virtio = 1 if $ds =~ m/^virtio/;
3549
3550         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3551
3552         if ($drive->{interface} eq 'virtio'){
3553            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3554         }
3555
3556         if ($drive->{interface} eq 'scsi') {
3557
3558             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3559
3560             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3561                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3562
3563             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3564             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3565
3566             my $iothread = '';
3567             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3568                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3569                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3570             } elsif ($drive->{iothread}) {
3571                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3572             }
3573
3574             my $queues = '';
3575             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3576                 $queues = ",num_queues=$drive->{queues}";
3577             }
3578
3579             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3580                 if !$scsicontroller->{$controller};
3581             $scsicontroller->{$controller}=1;
3582         }
3583
3584         if ($drive->{interface} eq 'sata') {
3585             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3586             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3587             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3588                 if !$ahcicontroller->{$controller};
3589             $ahcicontroller->{$controller}=1;
3590         }
3591
3592         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3593         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3594
3595         push @$devices, '-drive',$drive_cmd;
3596         push @$devices, '-device', print_drivedevice_full(
3597             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3598     });
3599
3600     for (my $i = 0; $i < $MAX_NETS; $i++) {
3601         my $netname = "net$i";
3602
3603         next if !$conf->{$netname};
3604         my $d = parse_net($conf->{$netname});
3605         next if !$d;
3606
3607         $use_virtio = 1 if $d->{model} eq 'virtio';
3608
3609         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3610
3611         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3612         push @$devices, '-netdev', $netdevfull;
3613
3614         my $netdevicefull = print_netdevice_full(
3615             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3616
3617         push @$devices, '-device', $netdevicefull;
3618     }
3619
3620     if ($conf->{ivshmem}) {
3621         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3622
3623         my $bus;
3624         if ($q35) {
3625             $bus = print_pcie_addr("ivshmem");
3626         } else {
3627             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3628         }
3629
3630         my $ivshmem_name = $ivshmem->{name} // $vmid;
3631         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3632
3633         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3634         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3635             .",size=$ivshmem->{size}M";
3636     }
3637
3638     # pci.4 is nested in pci.1
3639     $bridges->{1} = 1 if $bridges->{4};
3640
3641     if (!$q35) {
3642         # add pci bridges
3643         if (min_version($machine_version, 2, 3)) {
3644            $bridges->{1} = 1;
3645            $bridges->{2} = 1;
3646         }
3647
3648         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3649
3650     }
3651
3652     for my $k (sort {$b cmp $a} keys %$bridges) {
3653         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3654
3655         my $k_name = $k;
3656         if ($k == 2 && $legacy_igd) {
3657             $k_name = "$k-igd";
3658         }
3659         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3660
3661         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3662         if ($q35) {
3663             # add after -readconfig pve-q35.cfg
3664             splice @$devices, 2, 0, '-device', $devstr;
3665         } else {
3666             unshift @$devices, '-device', $devstr if $k > 0;
3667         }
3668     }
3669
3670     if (!$kvm) {
3671         push @$machineFlags, 'accel=tcg';
3672     }
3673
3674     my $machine_type_min = $machine_type;
3675     if ($add_pve_version) {
3676         $machine_type_min =~ s/\+pve\d+$//;
3677         $machine_type_min .= "+pve$required_pve_version";
3678     }
3679     push @$machineFlags, "type=${machine_type_min}";
3680
3681     push @$cmd, @$devices;
3682     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3683     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3684     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3685
3686     if (my $vmstate = $conf->{vmstate}) {
3687         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3688         push @$vollist, $vmstate;
3689         push @$cmd, '-loadstate', $statepath;
3690         print "activating and using '$vmstate' as vmstate\n";
3691     }
3692
3693     # add custom args
3694     if ($conf->{args}) {
3695         my $aa = PVE::Tools::split_args($conf->{args});
3696         push @$cmd, @$aa;
3697     }
3698
3699     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3700 }
3701
3702 sub check_rng_source {
3703     my ($source) = @_;
3704
3705     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3706     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3707         if ! -e $source;
3708
3709     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3710     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3711         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3712         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3713         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3714             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3715             ." to the host.\n";
3716     }
3717 }
3718
3719 sub spice_port {
3720     my ($vmid) = @_;
3721
3722     my $res = mon_cmd($vmid, 'query-spice');
3723
3724     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3725 }
3726
3727 sub vm_devices_list {
3728     my ($vmid) = @_;
3729
3730     my $res = mon_cmd($vmid, 'query-pci');
3731     my $devices_to_check = [];
3732     my $devices = {};
3733     foreach my $pcibus (@$res) {
3734         push @$devices_to_check, @{$pcibus->{devices}},
3735     }
3736
3737     while (@$devices_to_check) {
3738         my $to_check = [];
3739         for my $d (@$devices_to_check) {
3740             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3741             next if !$d->{'pci_bridge'};
3742
3743             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3744             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3745         }
3746         $devices_to_check = $to_check;
3747     }
3748
3749     my $resblock = mon_cmd($vmid, 'query-block');
3750     foreach my $block (@$resblock) {
3751         if($block->{device} =~ m/^drive-(\S+)/){
3752                 $devices->{$1} = 1;
3753         }
3754     }
3755
3756     my $resmice = mon_cmd($vmid, 'query-mice');
3757     foreach my $mice (@$resmice) {
3758         if ($mice->{name} eq 'QEMU HID Tablet') {
3759             $devices->{tablet} = 1;
3760             last;
3761         }
3762     }
3763
3764     # for usb devices there is no query-usb
3765     # but we can iterate over the entries in
3766     # qom-list path=/machine/peripheral
3767     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3768     foreach my $per (@$resperipheral) {
3769         if ($per->{name} =~ m/^usb\d+$/) {
3770             $devices->{$per->{name}} = 1;
3771         }
3772     }
3773
3774     return $devices;
3775 }
3776
3777 sub vm_deviceplug {
3778     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3779
3780     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3781
3782     my $devices_list = vm_devices_list($vmid);
3783     return 1 if defined($devices_list->{$deviceid});
3784
3785     # add PCI bridge if we need it for the device
3786     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3787
3788     if ($deviceid eq 'tablet') {
3789
3790         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3791
3792     } elsif ($deviceid eq 'keyboard') {
3793
3794         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3795
3796     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3797
3798         die "usb hotplug currently not reliable\n";
3799         # since we can't reliably hot unplug all added usb devices and usb
3800         # passthrough breaks live migration we disable usb hotplugging for now
3801         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3802
3803     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3804
3805         qemu_iothread_add($vmid, $deviceid, $device);
3806
3807         qemu_driveadd($storecfg, $vmid, $device);
3808         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3809
3810         qemu_deviceadd($vmid, $devicefull);
3811         eval { qemu_deviceaddverify($vmid, $deviceid); };
3812         if (my $err = $@) {
3813             eval { qemu_drivedel($vmid, $deviceid); };
3814             warn $@ if $@;
3815             die $err;
3816         }
3817
3818     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3819
3820
3821         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3822         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3823         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3824
3825         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3826
3827         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3828             qemu_iothread_add($vmid, $deviceid, $device);
3829             $devicefull .= ",iothread=iothread-$deviceid";
3830         }
3831
3832         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3833             $devicefull .= ",num_queues=$device->{queues}";
3834         }
3835
3836         qemu_deviceadd($vmid, $devicefull);
3837         qemu_deviceaddverify($vmid, $deviceid);
3838
3839     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3840
3841         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3842         qemu_driveadd($storecfg, $vmid, $device);
3843
3844         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3845         eval { qemu_deviceadd($vmid, $devicefull); };
3846         if (my $err = $@) {
3847             eval { qemu_drivedel($vmid, $deviceid); };
3848             warn $@ if $@;
3849             die $err;
3850         }
3851
3852     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3853
3854         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3855
3856         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3857         my $use_old_bios_files = undef;
3858         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3859
3860         my $netdevicefull = print_netdevice_full(
3861             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3862         qemu_deviceadd($vmid, $netdevicefull);
3863         eval {
3864             qemu_deviceaddverify($vmid, $deviceid);
3865             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3866         };
3867         if (my $err = $@) {
3868             eval { qemu_netdevdel($vmid, $deviceid); };
3869             warn $@ if $@;
3870             die $err;
3871         }
3872
3873     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3874
3875         my $bridgeid = $2;
3876         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3877         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3878
3879         qemu_deviceadd($vmid, $devicefull);
3880         qemu_deviceaddverify($vmid, $deviceid);
3881
3882     } else {
3883         die "can't hotplug device '$deviceid'\n";
3884     }
3885
3886     return 1;
3887 }
3888
3889 # fixme: this should raise exceptions on error!
3890 sub vm_deviceunplug {
3891     my ($vmid, $conf, $deviceid) = @_;
3892
3893     my $devices_list = vm_devices_list($vmid);
3894     return 1 if !defined($devices_list->{$deviceid});
3895
3896     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3897     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3898
3899     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3900
3901         qemu_devicedel($vmid, $deviceid);
3902
3903     } elsif ($deviceid =~ m/^usb\d+$/) {
3904
3905         die "usb hotplug currently not reliable\n";
3906         # when unplugging usb devices this way, there may be remaining usb
3907         # controllers/hubs so we disable it for now
3908         #qemu_devicedel($vmid, $deviceid);
3909         #qemu_devicedelverify($vmid, $deviceid);
3910
3911     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3912
3913         qemu_devicedel($vmid, $deviceid);
3914         qemu_devicedelverify($vmid, $deviceid);
3915         qemu_drivedel($vmid, $deviceid);
3916         qemu_iothread_del($conf, $vmid, $deviceid);
3917
3918     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3919
3920         qemu_devicedel($vmid, $deviceid);
3921         qemu_devicedelverify($vmid, $deviceid);
3922         qemu_iothread_del($conf, $vmid, $deviceid);
3923
3924     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3925
3926         qemu_devicedel($vmid, $deviceid);
3927         qemu_drivedel($vmid, $deviceid);
3928         qemu_deletescsihw($conf, $vmid, $deviceid);
3929
3930     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3931
3932         qemu_devicedel($vmid, $deviceid);
3933         qemu_devicedelverify($vmid, $deviceid);
3934         qemu_netdevdel($vmid, $deviceid);
3935
3936     } else {
3937         die "can't unplug device '$deviceid'\n";
3938     }
3939
3940     return 1;
3941 }
3942
3943 sub qemu_deviceadd {
3944     my ($vmid, $devicefull) = @_;
3945
3946     $devicefull = "driver=".$devicefull;
3947     my %options =  split(/[=,]/, $devicefull);
3948
3949     mon_cmd($vmid, "device_add" , %options);
3950 }
3951
3952 sub qemu_devicedel {
3953     my ($vmid, $deviceid) = @_;
3954
3955     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3956 }
3957
3958 sub qemu_iothread_add {
3959     my($vmid, $deviceid, $device) = @_;
3960
3961     if ($device->{iothread}) {
3962         my $iothreads = vm_iothreads_list($vmid);
3963         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3964     }
3965 }
3966
3967 sub qemu_iothread_del {
3968     my($conf, $vmid, $deviceid) = @_;
3969
3970     my $confid = $deviceid;
3971     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3972         $confid = 'scsi' . $1;
3973     }
3974     my $device = parse_drive($confid, $conf->{$confid});
3975     if ($device->{iothread}) {
3976         my $iothreads = vm_iothreads_list($vmid);
3977         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3978     }
3979 }
3980
3981 sub qemu_objectadd {
3982     my($vmid, $objectid, $qomtype) = @_;
3983
3984     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3985
3986     return 1;
3987 }
3988
3989 sub qemu_objectdel {
3990     my($vmid, $objectid) = @_;
3991
3992     mon_cmd($vmid, "object-del", id => $objectid);
3993
3994     return 1;
3995 }
3996
3997 sub qemu_driveadd {
3998     my ($storecfg, $vmid, $device) = @_;
3999
4000     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4001     $drive =~ s/\\/\\\\/g;
4002     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4003
4004     # If the command succeeds qemu prints: "OK"
4005     return 1 if $ret =~ m/OK/s;
4006
4007     die "adding drive failed: $ret\n";
4008 }
4009
4010 sub qemu_drivedel {
4011     my($vmid, $deviceid) = @_;
4012
4013     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4014     $ret =~ s/^\s+//;
4015
4016     return 1 if $ret eq "";
4017
4018     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4019     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4020
4021     die "deleting drive $deviceid failed : $ret\n";
4022 }
4023
4024 sub qemu_deviceaddverify {
4025     my ($vmid, $deviceid) = @_;
4026
4027     for (my $i = 0; $i <= 5; $i++) {
4028          my $devices_list = vm_devices_list($vmid);
4029          return 1 if defined($devices_list->{$deviceid});
4030          sleep 1;
4031     }
4032
4033     die "error on hotplug device '$deviceid'\n";
4034 }
4035
4036
4037 sub qemu_devicedelverify {
4038     my ($vmid, $deviceid) = @_;
4039
4040     # need to verify that the device is correctly removed as device_del
4041     # is async and empty return is not reliable
4042
4043     for (my $i = 0; $i <= 5; $i++) {
4044          my $devices_list = vm_devices_list($vmid);
4045          return 1 if !defined($devices_list->{$deviceid});
4046          sleep 1;
4047     }
4048
4049     die "error on hot-unplugging device '$deviceid'\n";
4050 }
4051
4052 sub qemu_findorcreatescsihw {
4053     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4054
4055     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4056
4057     my $scsihwid="$controller_prefix$controller";
4058     my $devices_list = vm_devices_list($vmid);
4059
4060     if(!defined($devices_list->{$scsihwid})) {
4061         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4062     }
4063
4064     return 1;
4065 }
4066
4067 sub qemu_deletescsihw {
4068     my ($conf, $vmid, $opt) = @_;
4069
4070     my $device = parse_drive($opt, $conf->{$opt});
4071
4072     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4073         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4074         return 1;
4075     }
4076
4077     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4078
4079     my $devices_list = vm_devices_list($vmid);
4080     foreach my $opt (keys %{$devices_list}) {
4081         if (is_valid_drivename($opt)) {
4082             my $drive = parse_drive($opt, $conf->{$opt});
4083             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4084                 return 1;
4085             }
4086         }
4087     }
4088
4089     my $scsihwid="scsihw$controller";
4090
4091     vm_deviceunplug($vmid, $conf, $scsihwid);
4092
4093     return 1;
4094 }
4095
4096 sub qemu_add_pci_bridge {
4097     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4098
4099     my $bridges = {};
4100
4101     my $bridgeid;
4102
4103     print_pci_addr($device, $bridges, $arch, $machine_type);
4104
4105     while (my ($k, $v) = each %$bridges) {
4106         $bridgeid = $k;
4107     }
4108     return 1 if !defined($bridgeid) || $bridgeid < 1;
4109
4110     my $bridge = "pci.$bridgeid";
4111     my $devices_list = vm_devices_list($vmid);
4112
4113     if (!defined($devices_list->{$bridge})) {
4114         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4115     }
4116
4117     return 1;
4118 }
4119
4120 sub qemu_set_link_status {
4121     my ($vmid, $device, $up) = @_;
4122
4123     mon_cmd($vmid, "set_link", name => $device,
4124                up => $up ? JSON::true : JSON::false);
4125 }
4126
4127 sub qemu_netdevadd {
4128     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4129
4130     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4131     my %options =  split(/[=,]/, $netdev);
4132
4133     if (defined(my $vhost = $options{vhost})) {
4134         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4135     }
4136
4137     if (defined(my $queues = $options{queues})) {
4138         $options{queues} = $queues + 0;
4139     }
4140
4141     mon_cmd($vmid, "netdev_add",  %options);
4142     return 1;
4143 }
4144
4145 sub qemu_netdevdel {
4146     my ($vmid, $deviceid) = @_;
4147
4148     mon_cmd($vmid, "netdev_del", id => $deviceid);
4149 }
4150
4151 sub qemu_usb_hotplug {
4152     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4153
4154     return if !$device;
4155
4156     # remove the old one first
4157     vm_deviceunplug($vmid, $conf, $deviceid);
4158
4159     # check if xhci controller is necessary and available
4160     if ($device->{usb3}) {
4161
4162         my $devicelist = vm_devices_list($vmid);
4163
4164         if (!$devicelist->{xhci}) {
4165             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4166             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4167         }
4168     }
4169     my $d = parse_usb_device($device->{host});
4170     $d->{usb3} = $device->{usb3};
4171
4172     # add the new one
4173     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4174 }
4175
4176 sub qemu_cpu_hotplug {
4177     my ($vmid, $conf, $vcpus) = @_;
4178
4179     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4180
4181     my $sockets = 1;
4182     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4183     $sockets = $conf->{sockets} if  $conf->{sockets};
4184     my $cores = $conf->{cores} || 1;
4185     my $maxcpus = $sockets * $cores;
4186
4187     $vcpus = $maxcpus if !$vcpus;
4188
4189     die "you can't add more vcpus than maxcpus\n"
4190         if $vcpus > $maxcpus;
4191
4192     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4193
4194     if ($vcpus < $currentvcpus) {
4195
4196         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4197
4198             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4199                 qemu_devicedel($vmid, "cpu$i");
4200                 my $retry = 0;
4201                 my $currentrunningvcpus = undef;
4202                 while (1) {
4203                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4204                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4205                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4206                     $retry++;
4207                     sleep 1;
4208                 }
4209                 #update conf after each succesfull cpu unplug
4210                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4211                 PVE::QemuConfig->write_config($vmid, $conf);
4212             }
4213         } else {
4214             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4215         }
4216
4217         return;
4218     }
4219
4220     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4221     die "vcpus in running vm does not match its configuration\n"
4222         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4223
4224     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4225
4226         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4227             my $cpustr = print_cpu_device($conf, $i);
4228             qemu_deviceadd($vmid, $cpustr);
4229
4230             my $retry = 0;
4231             my $currentrunningvcpus = undef;
4232             while (1) {
4233                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4234                 last if scalar(@{$currentrunningvcpus}) == $i;
4235                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4236                 sleep 1;
4237                 $retry++;
4238             }
4239             #update conf after each succesfull cpu hotplug
4240             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4241             PVE::QemuConfig->write_config($vmid, $conf);
4242         }
4243     } else {
4244
4245         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4246             mon_cmd($vmid, "cpu-add", id => int($i));
4247         }
4248     }
4249 }
4250
4251 sub qemu_block_set_io_throttle {
4252     my ($vmid, $deviceid,
4253         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4254         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4255         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4256         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4257
4258     return if !check_running($vmid) ;
4259
4260     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4261         bps => int($bps),
4262         bps_rd => int($bps_rd),
4263         bps_wr => int($bps_wr),
4264         iops => int($iops),
4265         iops_rd => int($iops_rd),
4266         iops_wr => int($iops_wr),
4267         bps_max => int($bps_max),
4268         bps_rd_max => int($bps_rd_max),
4269         bps_wr_max => int($bps_wr_max),
4270         iops_max => int($iops_max),
4271         iops_rd_max => int($iops_rd_max),
4272         iops_wr_max => int($iops_wr_max),
4273         bps_max_length => int($bps_max_length),
4274         bps_rd_max_length => int($bps_rd_max_length),
4275         bps_wr_max_length => int($bps_wr_max_length),
4276         iops_max_length => int($iops_max_length),
4277         iops_rd_max_length => int($iops_rd_max_length),
4278         iops_wr_max_length => int($iops_wr_max_length),
4279     );
4280
4281 }
4282
4283 sub qemu_block_resize {
4284     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4285
4286     my $running = check_running($vmid);
4287
4288     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4289
4290     return if !$running;
4291
4292     my $padding = (1024 - $size % 1024) % 1024;
4293     $size = $size + $padding;
4294
4295     mon_cmd(
4296         $vmid,
4297         "block_resize",
4298         device => $deviceid,
4299         size => int($size),
4300         timeout => 60,
4301     );
4302 }
4303
4304 sub qemu_volume_snapshot {
4305     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4306
4307     my $running = check_running($vmid);
4308
4309     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4310         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4311     } else {
4312         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4313     }
4314 }
4315
4316 sub qemu_volume_snapshot_delete {
4317     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4318
4319     my $running = check_running($vmid);
4320
4321     if($running) {
4322
4323         $running = undef;
4324         my $conf = PVE::QemuConfig->load_config($vmid);
4325         PVE::QemuConfig->foreach_volume($conf, sub {
4326             my ($ds, $drive) = @_;
4327             $running = 1 if $drive->{file} eq $volid;
4328         });
4329     }
4330
4331     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4332         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4333     } else {
4334         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4335     }
4336 }
4337
4338 sub set_migration_caps {
4339     my ($vmid, $savevm) = @_;
4340
4341     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4342
4343     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4344     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4345
4346     my $cap_ref = [];
4347
4348     my $enabled_cap = {
4349         "auto-converge" => 1,
4350         "xbzrle" => 1,
4351         "x-rdma-pin-all" => 0,
4352         "zero-blocks" => 0,
4353         "compress" => 0,
4354         "dirty-bitmaps" => $dirty_bitmaps,
4355     };
4356
4357     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4358
4359     for my $supported_capability (@$supported_capabilities) {
4360         push @$cap_ref, {
4361             capability => $supported_capability->{capability},
4362             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4363         };
4364     }
4365
4366     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4367 }
4368
4369 sub foreach_volid {
4370     my ($conf, $func, @param) = @_;
4371
4372     my $volhash = {};
4373
4374     my $test_volid = sub {
4375         my ($key, $drive, $snapname) = @_;
4376
4377         my $volid = $drive->{file};
4378         return if !$volid;
4379
4380         $volhash->{$volid}->{cdrom} //= 1;
4381         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4382
4383         my $replicate = $drive->{replicate} // 1;
4384         $volhash->{$volid}->{replicate} //= 0;
4385         $volhash->{$volid}->{replicate} = 1 if $replicate;
4386
4387         $volhash->{$volid}->{shared} //= 0;
4388         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4389
4390         $volhash->{$volid}->{referenced_in_config} //= 0;
4391         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4392
4393         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4394             if defined($snapname);
4395
4396         my $size = $drive->{size};
4397         $volhash->{$volid}->{size} //= $size if $size;
4398
4399         $volhash->{$volid}->{is_vmstate} //= 0;
4400         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4401
4402         $volhash->{$volid}->{is_unused} //= 0;
4403         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4404     };
4405
4406     my $include_opts = {
4407         extra_keys => ['vmstate'],
4408         include_unused => 1,
4409     };
4410
4411     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4412     foreach my $snapname (keys %{$conf->{snapshots}}) {
4413         my $snap = $conf->{snapshots}->{$snapname};
4414         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4415     }
4416
4417     foreach my $volid (keys %$volhash) {
4418         &$func($volid, $volhash->{$volid}, @param);
4419     }
4420 }
4421
4422 my $fast_plug_option = {
4423     'lock' => 1,
4424     'name' => 1,
4425     'onboot' => 1,
4426     'shares' => 1,
4427     'startup' => 1,
4428     'description' => 1,
4429     'protection' => 1,
4430     'vmstatestorage' => 1,
4431     'hookscript' => 1,
4432     'tags' => 1,
4433 };
4434
4435 # hotplug changes in [PENDING]
4436 # $selection hash can be used to only apply specified options, for
4437 # example: { cores => 1 } (only apply changed 'cores')
4438 # $errors ref is used to return error messages
4439 sub vmconfig_hotplug_pending {
4440     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4441
4442     my $defaults = load_defaults();
4443     my $arch = get_vm_arch($conf);
4444     my $machine_type = get_vm_machine($conf, undef, $arch);
4445
4446     # commit values which do not have any impact on running VM first
4447     # Note: those option cannot raise errors, we we do not care about
4448     # $selection and always apply them.
4449
4450     my $add_error = sub {
4451         my ($opt, $msg) = @_;
4452         $errors->{$opt} = "hotplug problem - $msg";
4453     };
4454
4455     my $changes = 0;
4456     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4457         if ($fast_plug_option->{$opt}) {
4458             $conf->{$opt} = $conf->{pending}->{$opt};
4459             delete $conf->{pending}->{$opt};
4460             $changes = 1;
4461         }
4462     }
4463
4464     if ($changes) {
4465         PVE::QemuConfig->write_config($vmid, $conf);
4466     }
4467
4468     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4469
4470     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4471     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4472     foreach my $opt (sort keys %$pending_delete_hash) {
4473         next if $selection && !$selection->{$opt};
4474         my $force = $pending_delete_hash->{$opt}->{force};
4475         eval {
4476             if ($opt eq 'hotplug') {
4477                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4478             } elsif ($opt eq 'tablet') {
4479                 die "skip\n" if !$hotplug_features->{usb};
4480                 if ($defaults->{tablet}) {
4481                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4482                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4483                         if $arch eq 'aarch64';
4484                 } else {
4485                     vm_deviceunplug($vmid, $conf, 'tablet');
4486                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4487                 }
4488             } elsif ($opt =~ m/^usb\d+/) {
4489                 die "skip\n";
4490                 # since we cannot reliably hot unplug usb devices we are disabling it
4491                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4492                 #vm_deviceunplug($vmid, $conf, $opt);
4493             } elsif ($opt eq 'vcpus') {
4494                 die "skip\n" if !$hotplug_features->{cpu};
4495                 qemu_cpu_hotplug($vmid, $conf, undef);
4496             } elsif ($opt eq 'balloon') {
4497                 # enable balloon device is not hotpluggable
4498                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4499                 # here we reset the ballooning value to memory
4500                 my $balloon = $conf->{memory} || $defaults->{memory};
4501                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4502             } elsif ($fast_plug_option->{$opt}) {
4503                 # do nothing
4504             } elsif ($opt =~ m/^net(\d+)$/) {
4505                 die "skip\n" if !$hotplug_features->{network};
4506                 vm_deviceunplug($vmid, $conf, $opt);
4507             } elsif (is_valid_drivename($opt)) {
4508                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4509                 vm_deviceunplug($vmid, $conf, $opt);
4510                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4511             } elsif ($opt =~ m/^memory$/) {
4512                 die "skip\n" if !$hotplug_features->{memory};
4513                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4514             } elsif ($opt eq 'cpuunits') {
4515                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4516             } elsif ($opt eq 'cpulimit') {
4517                 $cgroup->change_cpu_quota(-1, 100000);
4518             } else {
4519                 die "skip\n";
4520             }
4521         };
4522         if (my $err = $@) {
4523             &$add_error($opt, $err) if $err ne "skip\n";
4524         } else {
4525             delete $conf->{$opt};
4526             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4527         }
4528     }
4529
4530     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4531     $apply_pending_cloudinit = sub {
4532         return if $apply_pending_cloudinit_done; # once is enough
4533         $apply_pending_cloudinit_done = 1; # once is enough
4534
4535         my ($key, $value) = @_;
4536
4537         my @cloudinit_opts = keys %$confdesc_cloudinit;
4538         foreach my $opt (keys %{$conf->{pending}}) {
4539             next if !grep { $_ eq $opt } @cloudinit_opts;
4540             $conf->{$opt} = delete $conf->{pending}->{$opt};
4541         }
4542
4543         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4544         foreach my $opt (sort keys %$pending_delete_hash) {
4545             next if !grep { $_ eq $opt } @cloudinit_opts;
4546             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4547             delete $conf->{$opt};
4548         }
4549
4550         my $new_conf = { %$conf };
4551         $new_conf->{$key} = $value;
4552         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4553     };
4554
4555     foreach my $opt (keys %{$conf->{pending}}) {
4556         next if $selection && !$selection->{$opt};
4557         my $value = $conf->{pending}->{$opt};
4558         eval {
4559             if ($opt eq 'hotplug') {
4560                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4561             } elsif ($opt eq 'tablet') {
4562                 die "skip\n" if !$hotplug_features->{usb};
4563                 if ($value == 1) {
4564                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4565                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4566                         if $arch eq 'aarch64';
4567                 } elsif ($value == 0) {
4568                     vm_deviceunplug($vmid, $conf, 'tablet');
4569                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4570                 }
4571             } elsif ($opt =~ m/^usb\d+$/) {
4572                 die "skip\n";
4573                 # since we cannot reliably hot unplug usb devices we disable it for now
4574                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4575                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4576                 #die "skip\n" if !$d;
4577                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4578             } elsif ($opt eq 'vcpus') {
4579                 die "skip\n" if !$hotplug_features->{cpu};
4580                 qemu_cpu_hotplug($vmid, $conf, $value);
4581             } elsif ($opt eq 'balloon') {
4582                 # enable/disable balloning device is not hotpluggable
4583                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4584                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4585                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4586
4587                 # allow manual ballooning if shares is set to zero
4588                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4589                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4590                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4591                 }
4592             } elsif ($opt =~ m/^net(\d+)$/) {
4593                 # some changes can be done without hotplug
4594                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4595                                     $vmid, $opt, $value, $arch, $machine_type);
4596             } elsif (is_valid_drivename($opt)) {
4597                 die "skip\n" if $opt eq 'efidisk0';
4598                 # some changes can be done without hotplug
4599                 my $drive = parse_drive($opt, $value);
4600                 if (drive_is_cloudinit($drive)) {
4601                     &$apply_pending_cloudinit($opt, $value);
4602                 }
4603                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4604                                      $vmid, $opt, $value, $arch, $machine_type);
4605             } elsif ($opt =~ m/^memory$/) { #dimms
4606                 die "skip\n" if !$hotplug_features->{memory};
4607                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4608             } elsif ($opt eq 'cpuunits') {
4609                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4610             } elsif ($opt eq 'cpulimit') {
4611                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4612                 $cgroup->change_cpu_quota($cpulimit, 100000);
4613             } else {
4614                 die "skip\n";  # skip non-hot-pluggable options
4615             }
4616         };
4617         if (my $err = $@) {
4618             &$add_error($opt, $err) if $err ne "skip\n";
4619         } else {
4620             $conf->{$opt} = $value;
4621             delete $conf->{pending}->{$opt};
4622         }
4623     }
4624
4625     PVE::QemuConfig->write_config($vmid, $conf);
4626 }
4627
4628 sub try_deallocate_drive {
4629     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4630
4631     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4632         my $volid = $drive->{file};
4633         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4634             my $sid = PVE::Storage::parse_volume_id($volid);
4635             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4636
4637             # check if the disk is really unused
4638             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4639                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4640             PVE::Storage::vdisk_free($storecfg, $volid);
4641             return 1;
4642         } else {
4643             # If vm is not owner of this disk remove from config
4644             return 1;
4645         }
4646     }
4647
4648     return;
4649 }
4650
4651 sub vmconfig_delete_or_detach_drive {
4652     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4653
4654     my $drive = parse_drive($opt, $conf->{$opt});
4655
4656     my $rpcenv = PVE::RPCEnvironment::get();
4657     my $authuser = $rpcenv->get_user();
4658
4659     if ($force) {
4660         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4661         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4662     } else {
4663         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4664     }
4665 }
4666
4667
4668
4669 sub vmconfig_apply_pending {
4670     my ($vmid, $conf, $storecfg, $errors) = @_;
4671
4672     my $add_apply_error = sub {
4673         my ($opt, $msg) = @_;
4674         my $err_msg = "unable to apply pending change $opt : $msg";
4675         $errors->{$opt} = $err_msg;
4676         warn $err_msg;
4677     };
4678
4679     # cold plug
4680
4681     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4682     foreach my $opt (sort keys %$pending_delete_hash) {
4683         my $force = $pending_delete_hash->{$opt}->{force};
4684         eval {
4685             if ($opt =~ m/^unused/) {
4686                 die "internal error";
4687             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4688                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4689             }
4690         };
4691         if (my $err = $@) {
4692             $add_apply_error->($opt, $err);
4693         } else {
4694             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4695             delete $conf->{$opt};
4696         }
4697     }
4698
4699     PVE::QemuConfig->cleanup_pending($conf);
4700
4701     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4702         next if $opt eq 'delete'; # just to be sure
4703         eval {
4704             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4705                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4706             }
4707         };
4708         if (my $err = $@) {
4709             $add_apply_error->($opt, $err);
4710         } else {
4711             $conf->{$opt} = delete $conf->{pending}->{$opt};
4712         }
4713     }
4714
4715     # write all changes at once to avoid unnecessary i/o
4716     PVE::QemuConfig->write_config($vmid, $conf);
4717 }
4718
4719 sub vmconfig_update_net {
4720     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4721
4722     my $newnet = parse_net($value);
4723
4724     if ($conf->{$opt}) {
4725         my $oldnet = parse_net($conf->{$opt});
4726
4727         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4728             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4729             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4730             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4731
4732             # for non online change, we try to hot-unplug
4733             die "skip\n" if !$hotplug;
4734             vm_deviceunplug($vmid, $conf, $opt);
4735         } else {
4736
4737             die "internal error" if $opt !~ m/net(\d+)/;
4738             my $iface = "tap${vmid}i$1";
4739
4740             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4741                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4742                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4743                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4744                 PVE::Network::tap_unplug($iface);
4745
4746                 if ($have_sdn) {
4747                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4748                 } else {
4749                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4750                 }
4751             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4752                 # Rate can be applied on its own but any change above needs to
4753                 # include the rate in tap_plug since OVS resets everything.
4754                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4755             }
4756
4757             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4758                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4759             }
4760
4761             return 1;
4762         }
4763     }
4764
4765     if ($hotplug) {
4766         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4767     } else {
4768         die "skip\n";
4769     }
4770 }
4771
4772 sub vmconfig_update_disk {
4773     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4774
4775     my $drive = parse_drive($opt, $value);
4776
4777     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4778         my $media = $drive->{media} || 'disk';
4779         my $oldmedia = $old_drive->{media} || 'disk';
4780         die "unable to change media type\n" if $media ne $oldmedia;
4781
4782         if (!drive_is_cdrom($old_drive)) {
4783
4784             if ($drive->{file} ne $old_drive->{file}) {
4785
4786                 die "skip\n" if !$hotplug;
4787
4788                 # unplug and register as unused
4789                 vm_deviceunplug($vmid, $conf, $opt);
4790                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4791
4792             } else {
4793                 # update existing disk
4794
4795                 # skip non hotpluggable value
4796                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4797                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4798                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4799                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4800                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4801                     die "skip\n";
4802                 }
4803
4804                 # apply throttle
4805                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4806                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4807                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4808                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4809                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4810                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4811                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4812                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4813                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4814                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4815                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4816                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4817                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4818                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4819                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4820                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4821                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4822                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4823
4824                     qemu_block_set_io_throttle(
4825                         $vmid,"drive-$opt",
4826                         ($drive->{mbps} || 0)*1024*1024,
4827                         ($drive->{mbps_rd} || 0)*1024*1024,
4828                         ($drive->{mbps_wr} || 0)*1024*1024,
4829                          $drive->{iops} || 0,
4830                          $drive->{iops_rd} || 0,
4831                          $drive->{iops_wr} || 0,
4832                         ($drive->{mbps_max} || 0)*1024*1024,
4833                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4834                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4835                          $drive->{iops_max} || 0,
4836                          $drive->{iops_rd_max} || 0,
4837                          $drive->{iops_wr_max} || 0,
4838                          $drive->{bps_max_length} || 1,
4839                          $drive->{bps_rd_max_length} || 1,
4840                          $drive->{bps_wr_max_length} || 1,
4841                          $drive->{iops_max_length} || 1,
4842                          $drive->{iops_rd_max_length} || 1,
4843                          $drive->{iops_wr_max_length} || 1,
4844                     );
4845
4846                 }
4847
4848                 return 1;
4849             }
4850
4851         } else { # cdrom
4852
4853             if ($drive->{file} eq 'none') {
4854                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4855                 if (drive_is_cloudinit($old_drive)) {
4856                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4857                 }
4858             } else {
4859                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4860
4861                 # force eject if locked
4862                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4863
4864                 if ($path) {
4865                     mon_cmd($vmid, "blockdev-change-medium",
4866                         id => "$opt", filename => "$path");
4867                 }
4868             }
4869
4870             return 1;
4871         }
4872     }
4873
4874     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4875     # hotplug new disks
4876     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4877     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4878 }
4879
4880 # called in locked context by incoming migration
4881 sub vm_migrate_get_nbd_disks {
4882     my ($storecfg, $conf, $replicated_volumes) = @_;
4883
4884     my $local_volumes = {};
4885     PVE::QemuConfig->foreach_volume($conf, sub {
4886         my ($ds, $drive) = @_;
4887
4888         return if drive_is_cdrom($drive);
4889
4890         my $volid = $drive->{file};
4891
4892         return if !$volid;
4893
4894         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4895
4896         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4897         return if $scfg->{shared};
4898
4899         # replicated disks re-use existing state via bitmap
4900         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4901         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4902     });
4903     return $local_volumes;
4904 }
4905
4906 # called in locked context by incoming migration
4907 sub vm_migrate_alloc_nbd_disks {
4908     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4909
4910     my $format = undef;
4911
4912     my $nbd = {};
4913     foreach my $opt (sort keys %$source_volumes) {
4914         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4915
4916         if ($use_existing) {
4917             $nbd->{$opt}->{drivestr} = print_drive($drive);
4918             $nbd->{$opt}->{volid} = $volid;
4919             $nbd->{$opt}->{replicated} = 1;
4920             next;
4921         }
4922
4923         # If a remote storage is specified and the format of the original
4924         # volume is not available there, fall back to the default format.
4925         # Otherwise use the same format as the original.
4926         if (!$storagemap->{identity}) {
4927             $storeid = map_storage($storagemap, $storeid);
4928             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4929             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4930             my $fileFormat = qemu_img_format($scfg, $volname);
4931             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4932         } else {
4933             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4934             $format = qemu_img_format($scfg, $volname);
4935         }
4936
4937         my $size = $drive->{size} / 1024;
4938         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4939         my $newdrive = $drive;
4940         $newdrive->{format} = $format;
4941         $newdrive->{file} = $newvolid;
4942         my $drivestr = print_drive($newdrive);
4943         $nbd->{$opt}->{drivestr} = $drivestr;
4944         $nbd->{$opt}->{volid} = $newvolid;
4945     }
4946
4947     return $nbd;
4948 }
4949
4950 # see vm_start_nolock for parameters, additionally:
4951 # migrate_opts:
4952 #   storagemap = parsed storage map for allocating NBD disks
4953 sub vm_start {
4954     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4955
4956     return PVE::QemuConfig->lock_config($vmid, sub {
4957         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4958
4959         die "you can't start a vm if it's a template\n"
4960             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4961
4962         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4963         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4964
4965         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4966
4967         if ($has_backup_lock && $running) {
4968             # a backup is currently running, attempt to start the guest in the
4969             # existing QEMU instance
4970             return vm_resume($vmid);
4971         }
4972
4973         PVE::QemuConfig->check_lock($conf)
4974             if !($params->{skiplock} || $has_suspended_lock);
4975
4976         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4977
4978         die "VM $vmid already running\n" if $running;
4979
4980         if (my $storagemap = $migrate_opts->{storagemap}) {
4981             my $replicated = $migrate_opts->{replicated_volumes};
4982             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4983             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4984
4985             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4986                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4987             }
4988         }
4989
4990         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4991     });
4992 }
4993
4994
4995 # params:
4996 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4997 #   skiplock => 0/1, skip checking for config lock
4998 #   skiptemplate => 0/1, skip checking whether VM is template
4999 #   forcemachine => to force Qemu machine (rollback/migration)
5000 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5001 #   timeout => in seconds
5002 #   paused => start VM in paused state (backup)
5003 #   resume => resume from hibernation
5004 # migrate_opts:
5005 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5006 #   migratedfrom => source node
5007 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5008 #   network => CIDR of migration network
5009 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5010 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5011 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5012 sub vm_start_nolock {
5013     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5014
5015     my $statefile = $params->{statefile};
5016     my $resume = $params->{resume};
5017
5018     my $migratedfrom = $migrate_opts->{migratedfrom};
5019     my $migration_type = $migrate_opts->{type};
5020
5021     my $res = {};
5022
5023     # clean up leftover reboot request files
5024     eval { clear_reboot_request($vmid); };
5025     warn $@ if $@;
5026
5027     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5028         vmconfig_apply_pending($vmid, $conf, $storecfg);
5029         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5030     }
5031
5032     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5033
5034     my $defaults = load_defaults();
5035
5036     # set environment variable useful inside network script
5037     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5038
5039     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5040
5041     my $forcemachine = $params->{forcemachine};
5042     my $forcecpu = $params->{forcecpu};
5043     if ($resume) {
5044         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5045         $forcemachine = $conf->{runningmachine};
5046         $forcecpu = $conf->{runningcpu};
5047         print "Resuming suspended VM\n";
5048     }
5049
5050     my ($cmd, $vollist, $spice_port) =
5051         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5052
5053     my $migration_ip;
5054     my $get_migration_ip = sub {
5055         my ($nodename) = @_;
5056
5057         return $migration_ip if defined($migration_ip);
5058
5059         my $cidr = $migrate_opts->{network};
5060
5061         if (!defined($cidr)) {
5062             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5063             $cidr = $dc_conf->{migration}->{network};
5064         }
5065
5066         if (defined($cidr)) {
5067             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5068
5069             die "could not get IP: no address configured on local " .
5070                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5071
5072             die "could not get IP: multiple addresses configured on local " .
5073                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5074
5075             $migration_ip = @$ips[0];
5076         }
5077
5078         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5079             if !defined($migration_ip);
5080
5081         return $migration_ip;
5082     };
5083
5084     my $migrate_uri;
5085     if ($statefile) {
5086         if ($statefile eq 'tcp') {
5087             my $localip = "localhost";
5088             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5089             my $nodename = nodename();
5090
5091             if (!defined($migration_type)) {
5092                 if (defined($datacenterconf->{migration}->{type})) {
5093                     $migration_type = $datacenterconf->{migration}->{type};
5094                 } else {
5095                     $migration_type = 'secure';
5096                 }
5097             }
5098
5099             if ($migration_type eq 'insecure') {
5100                 $localip = $get_migration_ip->($nodename);
5101                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5102             }
5103
5104             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5105             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5106             $migrate_uri = "tcp:${localip}:${migrate_port}";
5107             push @$cmd, '-incoming', $migrate_uri;
5108             push @$cmd, '-S';
5109
5110         } elsif ($statefile eq 'unix') {
5111             # should be default for secure migrations as a ssh TCP forward
5112             # tunnel is not deterministic reliable ready and fails regurarly
5113             # to set up in time, so use UNIX socket forwards
5114             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5115             unlink $socket_addr;
5116
5117             $migrate_uri = "unix:$socket_addr";
5118
5119             push @$cmd, '-incoming', $migrate_uri;
5120             push @$cmd, '-S';
5121
5122         } elsif (-e $statefile) {
5123             push @$cmd, '-loadstate', $statefile;
5124         } else {
5125             my $statepath = PVE::Storage::path($storecfg, $statefile);
5126             push @$vollist, $statefile;
5127             push @$cmd, '-loadstate', $statepath;
5128         }
5129     } elsif ($params->{paused}) {
5130         push @$cmd, '-S';
5131     }
5132
5133     # host pci devices
5134     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5135       my $d = parse_hostpci($conf->{"hostpci$i"});
5136       next if !$d;
5137       my $pcidevices = $d->{pciid};
5138       foreach my $pcidevice (@$pcidevices) {
5139             my $pciid = $pcidevice->{id};
5140
5141             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5142             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5143             die "no pci device info for device '$pciid'\n" if !$info;
5144
5145             if ($d->{mdev}) {
5146                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5147                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5148             } else {
5149                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5150                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5151                 die "can't reset PCI device '$pciid'\n"
5152                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5153             }
5154       }
5155     }
5156
5157     PVE::Storage::activate_volumes($storecfg, $vollist);
5158
5159     eval {
5160         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5161             outfunc => sub {}, errfunc => sub {});
5162     };
5163     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5164     # timeout should be more than enough here...
5165     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5166
5167     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5168                                               : $defaults->{cpuunits};
5169
5170     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5171     my %run_params = (
5172         timeout => $statefile ? undef : $start_timeout,
5173         umask => 0077,
5174         noerr => 1,
5175     );
5176
5177     # when migrating, prefix QEMU output so other side can pick up any
5178     # errors that might occur and show the user
5179     if ($migratedfrom) {
5180         $run_params{quiet} = 1;
5181         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5182     }
5183
5184     my %properties = (
5185         Slice => 'qemu.slice',
5186         KillMode => 'none'
5187     );
5188
5189     if (PVE::CGroup::cgroup_mode() == 2) {
5190         $properties{CPUWeight} = $cpuunits;
5191     } else {
5192         $properties{CPUShares} = $cpuunits;
5193     }
5194
5195     if (my $cpulimit = $conf->{cpulimit}) {
5196         $properties{CPUQuota} = int($cpulimit * 100);
5197     }
5198     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5199
5200     my $run_qemu = sub {
5201         PVE::Tools::run_fork sub {
5202             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5203
5204             my $exitcode = run_command($cmd, %run_params);
5205             die "QEMU exited with code $exitcode\n" if $exitcode;
5206         };
5207     };
5208
5209     if ($conf->{hugepages}) {
5210
5211         my $code = sub {
5212             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5213             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5214
5215             PVE::QemuServer::Memory::hugepages_mount();
5216             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5217
5218             eval { $run_qemu->() };
5219             if (my $err = $@) {
5220                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5221                     if !$conf->{keephugepages};
5222                 die $err;
5223             }
5224
5225             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5226                 if !$conf->{keephugepages};
5227         };
5228         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5229
5230     } else {
5231         eval { $run_qemu->() };
5232     }
5233
5234     if (my $err = $@) {
5235         # deactivate volumes if start fails
5236         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5237         die "start failed: $err";
5238     }
5239
5240     print "migration listens on $migrate_uri\n" if $migrate_uri;
5241     $res->{migrate_uri} = $migrate_uri;
5242
5243     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5244         eval { mon_cmd($vmid, "cont"); };
5245         warn $@ if $@;
5246     }
5247
5248     #start nbd server for storage migration
5249     if (my $nbd = $migrate_opts->{nbd}) {
5250         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5251
5252         my $migrate_storage_uri;
5253         # nbd_protocol_version > 0 for unix socket support
5254         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5255             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5256             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5257             $migrate_storage_uri = "nbd:unix:$socket_path";
5258         } else {
5259             my $nodename = nodename();
5260             my $localip = $get_migration_ip->($nodename);
5261             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5262             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5263
5264             mon_cmd($vmid, "nbd-server-start", addr => {
5265                 type => 'inet',
5266                 data => {
5267                     host => "${localip}",
5268                     port => "${storage_migrate_port}",
5269                 },
5270             });
5271             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5272             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5273         }
5274
5275         $res->{migrate_storage_uri} = $migrate_storage_uri;
5276
5277         foreach my $opt (sort keys %$nbd) {
5278             my $drivestr = $nbd->{$opt}->{drivestr};
5279             my $volid = $nbd->{$opt}->{volid};
5280             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5281             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5282             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5283             print "re-using replicated volume: $opt - $volid\n"
5284                 if $nbd->{$opt}->{replicated};
5285
5286             $res->{drives}->{$opt} = $nbd->{$opt};
5287             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5288         }
5289     }
5290
5291     if ($migratedfrom) {
5292         eval {
5293             set_migration_caps($vmid);
5294         };
5295         warn $@ if $@;
5296
5297         if ($spice_port) {
5298             print "spice listens on port $spice_port\n";
5299             $res->{spice_port} = $spice_port;
5300             if ($migrate_opts->{spice_ticket}) {
5301                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5302                     $migrate_opts->{spice_ticket});
5303                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5304             }
5305         }
5306
5307     } else {
5308         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5309             if !$statefile && $conf->{balloon};
5310
5311         foreach my $opt (keys %$conf) {
5312             next if $opt !~  m/^net\d+$/;
5313             my $nicconf = parse_net($conf->{$opt});
5314             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5315         }
5316     }
5317
5318     mon_cmd($vmid, 'qom-set',
5319                 path => "machine/peripheral/balloon0",
5320                 property => "guest-stats-polling-interval",
5321                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5322
5323     if ($resume) {
5324         print "Resumed VM, removing state\n";
5325         if (my $vmstate = $conf->{vmstate}) {
5326             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5327             PVE::Storage::vdisk_free($storecfg, $vmstate);
5328         }
5329         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5330         PVE::QemuConfig->write_config($vmid, $conf);
5331     }
5332
5333     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5334
5335     return $res;
5336 }
5337
5338 sub vm_commandline {
5339     my ($storecfg, $vmid, $snapname) = @_;
5340
5341     my $conf = PVE::QemuConfig->load_config($vmid);
5342     my $forcemachine;
5343     my $forcecpu;
5344
5345     if ($snapname) {
5346         my $snapshot = $conf->{snapshots}->{$snapname};
5347         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5348
5349         # check for machine or CPU overrides in snapshot
5350         $forcemachine = $snapshot->{runningmachine};
5351         $forcecpu = $snapshot->{runningcpu};
5352
5353         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5354
5355         $conf = $snapshot;
5356     }
5357
5358     my $defaults = load_defaults();
5359
5360     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5361         $forcemachine, $forcecpu);
5362
5363     return PVE::Tools::cmd2string($cmd);
5364 }
5365
5366 sub vm_reset {
5367     my ($vmid, $skiplock) = @_;
5368
5369     PVE::QemuConfig->lock_config($vmid, sub {
5370
5371         my $conf = PVE::QemuConfig->load_config($vmid);
5372
5373         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5374
5375         mon_cmd($vmid, "system_reset");
5376     });
5377 }
5378
5379 sub get_vm_volumes {
5380     my ($conf) = @_;
5381
5382     my $vollist = [];
5383     foreach_volid($conf, sub {
5384         my ($volid, $attr) = @_;
5385
5386         return if $volid =~ m|^/|;
5387
5388         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5389         return if !$sid;
5390
5391         push @$vollist, $volid;
5392     });
5393
5394     return $vollist;
5395 }
5396
5397 sub vm_stop_cleanup {
5398     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5399
5400     eval {
5401
5402         if (!$keepActive) {
5403             my $vollist = get_vm_volumes($conf);
5404             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5405         }
5406
5407         foreach my $ext (qw(mon qmp pid vnc qga)) {
5408             unlink "/var/run/qemu-server/${vmid}.$ext";
5409         }
5410
5411         if ($conf->{ivshmem}) {
5412             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5413             # just delete it for now, VMs which have this already open do not
5414             # are affected, but new VMs will get a separated one. If this
5415             # becomes an issue we either add some sort of ref-counting or just
5416             # add a "don't delete on stop" flag to the ivshmem format.
5417             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5418         }
5419
5420         foreach my $key (keys %$conf) {
5421             next if $key !~ m/^hostpci(\d+)$/;
5422             my $hostpciindex = $1;
5423             my $d = parse_hostpci($conf->{$key});
5424             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5425
5426             foreach my $pci (@{$d->{pciid}}) {
5427                 my $pciid = $pci->{id};
5428                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5429             }
5430         }
5431
5432         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5433     };
5434     warn $@ if $@; # avoid errors - just warn
5435 }
5436
5437 # call only in locked context
5438 sub _do_vm_stop {
5439     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5440
5441     my $pid = check_running($vmid, $nocheck);
5442     return if !$pid;
5443
5444     my $conf;
5445     if (!$nocheck) {
5446         $conf = PVE::QemuConfig->load_config($vmid);
5447         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5448         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5449             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5450             $timeout = $opts->{down} if $opts->{down};
5451         }
5452         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5453     }
5454
5455     eval {
5456         if ($shutdown) {
5457             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5458                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5459             } else {
5460                 mon_cmd($vmid, "system_powerdown");
5461             }
5462         } else {
5463             mon_cmd($vmid, "quit");
5464         }
5465     };
5466     my $err = $@;
5467
5468     if (!$err) {
5469         $timeout = 60 if !defined($timeout);
5470
5471         my $count = 0;
5472         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5473             $count++;
5474             sleep 1;
5475         }
5476
5477         if ($count >= $timeout) {
5478             if ($force) {
5479                 warn "VM still running - terminating now with SIGTERM\n";
5480                 kill 15, $pid;
5481             } else {
5482                 die "VM quit/powerdown failed - got timeout\n";
5483             }
5484         } else {
5485             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5486             return;
5487         }
5488     } else {
5489        if (!check_running($vmid, $nocheck)) {
5490             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5491             return;
5492        }
5493        if ($force) {
5494             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5495             kill 15, $pid;
5496         } else {
5497             die "VM quit/powerdown failed\n";
5498         }
5499     }
5500
5501     # wait again
5502     $timeout = 10;
5503
5504     my $count = 0;
5505     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5506         $count++;
5507         sleep 1;
5508     }
5509
5510     if ($count >= $timeout) {
5511         warn "VM still running - terminating now with SIGKILL\n";
5512         kill 9, $pid;
5513         sleep 1;
5514     }
5515
5516     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5517 }
5518
5519 # Note: use $nocheck to skip tests if VM configuration file exists.
5520 # We need that when migration VMs to other nodes (files already moved)
5521 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5522 sub vm_stop {
5523     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5524
5525     $force = 1 if !defined($force) && !$shutdown;
5526
5527     if ($migratedfrom){
5528         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5529         kill 15, $pid if $pid;
5530         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5531         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5532         return;
5533     }
5534
5535     PVE::QemuConfig->lock_config($vmid, sub {
5536         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5537    });
5538 }
5539
5540 sub vm_reboot {
5541     my ($vmid, $timeout) = @_;
5542
5543     PVE::QemuConfig->lock_config($vmid, sub {
5544         eval {
5545
5546             # only reboot if running, as qmeventd starts it again on a stop event
5547             return if !check_running($vmid);
5548
5549             create_reboot_request($vmid);
5550
5551             my $storecfg = PVE::Storage::config();
5552             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5553
5554         };
5555         if (my $err = $@) {
5556             # avoid that the next normal shutdown will be confused for a reboot
5557             clear_reboot_request($vmid);
5558             die $err;
5559         }
5560    });
5561 }
5562
5563 # note: if using the statestorage parameter, the caller has to check privileges
5564 sub vm_suspend {
5565     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5566
5567     my $conf;
5568     my $path;
5569     my $storecfg;
5570     my $vmstate;
5571
5572     PVE::QemuConfig->lock_config($vmid, sub {
5573
5574         $conf = PVE::QemuConfig->load_config($vmid);
5575
5576         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5577         PVE::QemuConfig->check_lock($conf)
5578             if !($skiplock || $is_backing_up);
5579
5580         die "cannot suspend to disk during backup\n"
5581             if $is_backing_up && $includestate;
5582
5583         if ($includestate) {
5584             $conf->{lock} = 'suspending';
5585             my $date = strftime("%Y-%m-%d", localtime(time()));
5586             $storecfg = PVE::Storage::config();
5587             if (!$statestorage) {
5588                 $statestorage = find_vmstate_storage($conf, $storecfg);
5589                 # check permissions for the storage
5590                 my $rpcenv = PVE::RPCEnvironment::get();
5591                 if ($rpcenv->{type} ne 'cli') {
5592                     my $authuser = $rpcenv->get_user();
5593                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5594                 }
5595             }
5596
5597
5598             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5599                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5600             $path = PVE::Storage::path($storecfg, $vmstate);
5601             PVE::QemuConfig->write_config($vmid, $conf);
5602         } else {
5603             mon_cmd($vmid, "stop");
5604         }
5605     });
5606
5607     if ($includestate) {
5608         # save vm state
5609         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5610
5611         eval {
5612             set_migration_caps($vmid, 1);
5613             mon_cmd($vmid, "savevm-start", statefile => $path);
5614             for(;;) {
5615                 my $state = mon_cmd($vmid, "query-savevm");
5616                 if (!$state->{status}) {
5617                     die "savevm not active\n";
5618                 } elsif ($state->{status} eq 'active') {
5619                     sleep(1);
5620                     next;
5621                 } elsif ($state->{status} eq 'completed') {
5622                     print "State saved, quitting\n";
5623                     last;
5624                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5625                     die "query-savevm failed with error '$state->{error}'\n"
5626                 } else {
5627                     die "query-savevm returned status '$state->{status}'\n";
5628                 }
5629             }
5630         };
5631         my $err = $@;
5632
5633         PVE::QemuConfig->lock_config($vmid, sub {
5634             $conf = PVE::QemuConfig->load_config($vmid);
5635             if ($err) {
5636                 # cleanup, but leave suspending lock, to indicate something went wrong
5637                 eval {
5638                     mon_cmd($vmid, "savevm-end");
5639                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5640                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5641                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5642                     PVE::QemuConfig->write_config($vmid, $conf);
5643                 };
5644                 warn $@ if $@;
5645                 die $err;
5646             }
5647
5648             die "lock changed unexpectedly\n"
5649                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5650
5651             mon_cmd($vmid, "quit");
5652             $conf->{lock} = 'suspended';
5653             PVE::QemuConfig->write_config($vmid, $conf);
5654         });
5655     }
5656 }
5657
5658 sub vm_resume {
5659     my ($vmid, $skiplock, $nocheck) = @_;
5660
5661     PVE::QemuConfig->lock_config($vmid, sub {
5662         my $res = mon_cmd($vmid, 'query-status');
5663         my $resume_cmd = 'cont';
5664         my $reset = 0;
5665
5666         if ($res->{status}) {
5667             return if $res->{status} eq 'running'; # job done, go home
5668             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5669             $reset = 1 if $res->{status} eq 'shutdown';
5670         }
5671
5672         if (!$nocheck) {
5673
5674             my $conf = PVE::QemuConfig->load_config($vmid);
5675
5676             PVE::QemuConfig->check_lock($conf)
5677                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5678         }
5679
5680         if ($reset) {
5681             # required if a VM shuts down during a backup and we get a resume
5682             # request before the backup finishes for example
5683             mon_cmd($vmid, "system_reset");
5684         }
5685         mon_cmd($vmid, $resume_cmd);
5686     });
5687 }
5688
5689 sub vm_sendkey {
5690     my ($vmid, $skiplock, $key) = @_;
5691
5692     PVE::QemuConfig->lock_config($vmid, sub {
5693
5694         my $conf = PVE::QemuConfig->load_config($vmid);
5695
5696         # there is no qmp command, so we use the human monitor command
5697         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5698         die $res if $res ne '';
5699     });
5700 }
5701
5702 # vzdump restore implementaion
5703
5704 sub tar_archive_read_firstfile {
5705     my $archive = shift;
5706
5707     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5708
5709     # try to detect archive type first
5710     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5711         die "unable to open file '$archive'\n";
5712     my $firstfile = <$fh>;
5713     kill 15, $pid;
5714     close $fh;
5715
5716     die "ERROR: archive contaions no data\n" if !$firstfile;
5717     chomp $firstfile;
5718
5719     return $firstfile;
5720 }
5721
5722 sub tar_restore_cleanup {
5723     my ($storecfg, $statfile) = @_;
5724
5725     print STDERR "starting cleanup\n";
5726
5727     if (my $fd = IO::File->new($statfile, "r")) {
5728         while (defined(my $line = <$fd>)) {
5729             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5730                 my $volid = $2;
5731                 eval {
5732                     if ($volid =~ m|^/|) {
5733                         unlink $volid || die 'unlink failed\n';
5734                     } else {
5735                         PVE::Storage::vdisk_free($storecfg, $volid);
5736                     }
5737                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5738                 };
5739                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5740             } else {
5741                 print STDERR "unable to parse line in statfile - $line";
5742             }
5743         }
5744         $fd->close();
5745     }
5746 }
5747
5748 sub restore_file_archive {
5749     my ($archive, $vmid, $user, $opts) = @_;
5750
5751     return restore_vma_archive($archive, $vmid, $user, $opts)
5752         if $archive eq '-';
5753
5754     my $info = PVE::Storage::archive_info($archive);
5755     my $format = $opts->{format} // $info->{format};
5756     my $comp = $info->{compression};
5757
5758     # try to detect archive format
5759     if ($format eq 'tar') {
5760         return restore_tar_archive($archive, $vmid, $user, $opts);
5761     } else {
5762         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5763     }
5764 }
5765
5766 # hepler to remove disks that will not be used after restore
5767 my $restore_cleanup_oldconf = sub {
5768     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5769
5770     PVE::QemuConfig->foreach_volume($oldconf, sub {
5771         my ($ds, $drive) = @_;
5772
5773         return if drive_is_cdrom($drive, 1);
5774
5775         my $volid = $drive->{file};
5776         return if !$volid || $volid =~ m|^/|;
5777
5778         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5779         return if !$path || !$owner || ($owner != $vmid);
5780
5781         # Note: only delete disk we want to restore
5782         # other volumes will become unused
5783         if ($virtdev_hash->{$ds}) {
5784             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5785             if (my $err = $@) {
5786                 warn $err;
5787             }
5788         }
5789     });
5790
5791     # delete vmstate files, after the restore we have no snapshots anymore
5792     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5793         my $snap = $oldconf->{snapshots}->{$snapname};
5794         if ($snap->{vmstate}) {
5795             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5796             if (my $err = $@) {
5797                 warn $err;
5798             }
5799         }
5800     }
5801 };
5802
5803 # Helper to parse vzdump backup device hints
5804 #
5805 # $rpcenv: Environment, used to ckeck storage permissions
5806 # $user: User ID, to check storage permissions
5807 # $storecfg: Storage configuration
5808 # $fh: the file handle for reading the configuration
5809 # $devinfo: should contain device sizes for all backu-up'ed devices
5810 # $options: backup options (pool, default storage)
5811 #
5812 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5813 my $parse_backup_hints = sub {
5814     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5815
5816     my $virtdev_hash = {};
5817
5818     while (defined(my $line = <$fh>)) {
5819         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5820             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5821             die "archive does not contain data for drive '$virtdev'\n"
5822                 if !$devinfo->{$devname};
5823
5824             if (defined($options->{storage})) {
5825                 $storeid = $options->{storage} || 'local';
5826             } elsif (!$storeid) {
5827                 $storeid = 'local';
5828             }
5829             $format = 'raw' if !$format;
5830             $devinfo->{$devname}->{devname} = $devname;
5831             $devinfo->{$devname}->{virtdev} = $virtdev;
5832             $devinfo->{$devname}->{format} = $format;
5833             $devinfo->{$devname}->{storeid} = $storeid;
5834
5835             # check permission on storage
5836             my $pool = $options->{pool}; # todo: do we need that?
5837             if ($user ne 'root@pam') {
5838                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5839             }
5840
5841             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5842         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5843             my $virtdev = $1;
5844             my $drive = parse_drive($virtdev, $2);
5845             if (drive_is_cloudinit($drive)) {
5846                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5847                 $storeid = $options->{storage} if defined ($options->{storage});
5848                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5849                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5850
5851                 $virtdev_hash->{$virtdev} = {
5852                     format => $format,
5853                     storeid => $storeid,
5854                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5855                     is_cloudinit => 1,
5856                 };
5857             }
5858         }
5859     }
5860
5861     return $virtdev_hash;
5862 };
5863
5864 # Helper to allocate and activate all volumes required for a restore
5865 #
5866 # $storecfg: Storage configuration
5867 # $virtdev_hash: as returned by parse_backup_hints()
5868 #
5869 # Returns: { $virtdev => $volid }
5870 my $restore_allocate_devices = sub {
5871     my ($storecfg, $virtdev_hash, $vmid) = @_;
5872
5873     my $map = {};
5874     foreach my $virtdev (sort keys %$virtdev_hash) {
5875         my $d = $virtdev_hash->{$virtdev};
5876         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5877         my $storeid = $d->{storeid};
5878         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5879
5880         # test if requested format is supported
5881         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5882         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5883         $d->{format} = $defFormat if !$supported;
5884
5885         my $name;
5886         if ($d->{is_cloudinit}) {
5887             $name = "vm-$vmid-cloudinit";
5888             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5889             if ($scfg->{path}) {
5890                 $name .= ".$d->{format}";
5891             }
5892         }
5893
5894         my $volid = PVE::Storage::vdisk_alloc(
5895             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5896
5897         print STDERR "new volume ID is '$volid'\n";
5898         $d->{volid} = $volid;
5899
5900         PVE::Storage::activate_volumes($storecfg, [$volid]);
5901
5902         $map->{$virtdev} = $volid;
5903     }
5904
5905     return $map;
5906 };
5907
5908 my $restore_update_config_line = sub {
5909     my ($cookie, $vmid, $map, $line, $unique) = @_;
5910
5911     return '' if $line =~ m/^\#qmdump\#/;
5912     return '' if $line =~ m/^\#vzdump\#/;
5913     return '' if $line =~ m/^lock:/;
5914     return '' if $line =~ m/^unused\d+:/;
5915     return '' if $line =~ m/^parent:/;
5916
5917     my $res = '';
5918
5919     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5920     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5921         # try to convert old 1.X settings
5922         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5923         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5924             my ($model, $macaddr) = split(/\=/, $devconfig);
5925             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5926             my $net = {
5927                 model => $model,
5928                 bridge => "vmbr$ind",
5929                 macaddr => $macaddr,
5930             };
5931             my $netstr = print_net($net);
5932
5933             $res .= "net$cookie->{netcount}: $netstr\n";
5934             $cookie->{netcount}++;
5935         }
5936     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5937         my ($id, $netstr) = ($1, $2);
5938         my $net = parse_net($netstr);
5939         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5940         $netstr = print_net($net);
5941         $res .= "$id: $netstr\n";
5942     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5943         my $virtdev = $1;
5944         my $value = $3;
5945         my $di = parse_drive($virtdev, $value);
5946         if (defined($di->{backup}) && !$di->{backup}) {
5947             $res .= "#$line";
5948         } elsif ($map->{$virtdev}) {
5949             delete $di->{format}; # format can change on restore
5950             $di->{file} = $map->{$virtdev};
5951             $value = print_drive($di);
5952             $res .= "$virtdev: $value\n";
5953         } else {
5954             $res .= $line;
5955         }
5956     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5957         my $vmgenid = $1;
5958         if ($vmgenid ne '0') {
5959             # always generate a new vmgenid if there was a valid one setup
5960             $vmgenid = generate_uuid();
5961         }
5962         $res .= "vmgenid: $vmgenid\n";
5963     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5964         my ($uuid, $uuid_str);
5965         UUID::generate($uuid);
5966         UUID::unparse($uuid, $uuid_str);
5967         my $smbios1 = parse_smbios1($2);
5968         $smbios1->{uuid} = $uuid_str;
5969         $res .= $1.print_smbios1($smbios1)."\n";
5970     } else {
5971         $res .= $line;
5972     }
5973
5974     return $res;
5975 };
5976
5977 my $restore_deactivate_volumes = sub {
5978     my ($storecfg, $devinfo) = @_;
5979
5980     my $vollist = [];
5981     foreach my $devname (keys %$devinfo) {
5982         my $volid = $devinfo->{$devname}->{volid};
5983         push @$vollist, $volid if $volid;
5984     }
5985
5986     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5987 };
5988
5989 my $restore_destroy_volumes = sub {
5990     my ($storecfg, $devinfo) = @_;
5991
5992     foreach my $devname (keys %$devinfo) {
5993         my $volid = $devinfo->{$devname}->{volid};
5994         next if !$volid;
5995         eval {
5996             if ($volid =~ m|^/|) {
5997                 unlink $volid || die 'unlink failed\n';
5998             } else {
5999                 PVE::Storage::vdisk_free($storecfg, $volid);
6000             }
6001             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6002         };
6003         print STDERR "unable to cleanup '$volid' - $@" if $@;
6004     }
6005 };
6006
6007 sub scan_volids {
6008     my ($cfg, $vmid) = @_;
6009
6010     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6011
6012     my $volid_hash = {};
6013     foreach my $storeid (keys %$info) {
6014         foreach my $item (@{$info->{$storeid}}) {
6015             next if !($item->{volid} && $item->{size});
6016             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6017             $volid_hash->{$item->{volid}} = $item;
6018         }
6019     }
6020
6021     return $volid_hash;
6022 }
6023
6024 sub update_disk_config {
6025     my ($vmid, $conf, $volid_hash) = @_;
6026
6027     my $changes;
6028     my $prefix = "VM $vmid";
6029
6030     # used and unused disks
6031     my $referenced = {};
6032
6033     # Note: it is allowed to define multiple storages with same path (alias), so
6034     # we need to check both 'volid' and real 'path' (two different volid can point
6035     # to the same path).
6036
6037     my $referencedpath = {};
6038
6039     # update size info
6040     PVE::QemuConfig->foreach_volume($conf, sub {
6041         my ($opt, $drive) = @_;
6042
6043         my $volid = $drive->{file};
6044         return if !$volid;
6045         my $volume = $volid_hash->{$volid};
6046
6047         # mark volid as "in-use" for next step
6048         $referenced->{$volid} = 1;
6049         if ($volume && (my $path = $volume->{path})) {
6050             $referencedpath->{$path} = 1;
6051         }
6052
6053         return if drive_is_cdrom($drive);
6054         return if !$volume;
6055
6056         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6057         if (defined($updated)) {
6058             $changes = 1;
6059             $conf->{$opt} = print_drive($updated);
6060             print "$prefix ($opt): $msg\n";
6061         }
6062     });
6063
6064     # remove 'unusedX' entry if volume is used
6065     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6066         my ($opt, $drive) = @_;
6067
6068         my $volid = $drive->{file};
6069         return if !$volid;
6070
6071         my $path;
6072         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6073         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6074             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6075             $changes = 1;
6076             delete $conf->{$opt};
6077         }
6078
6079         $referenced->{$volid} = 1;
6080         $referencedpath->{$path} = 1 if $path;
6081     });
6082
6083     foreach my $volid (sort keys %$volid_hash) {
6084         next if $volid =~ m/vm-$vmid-state-/;
6085         next if $referenced->{$volid};
6086         my $path = $volid_hash->{$volid}->{path};
6087         next if !$path; # just to be sure
6088         next if $referencedpath->{$path};
6089         $changes = 1;
6090         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6091         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6092         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6093     }
6094
6095     return $changes;
6096 }
6097
6098 sub rescan {
6099     my ($vmid, $nolock, $dryrun) = @_;
6100
6101     my $cfg = PVE::Storage::config();
6102
6103     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6104     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6105     foreach my $stor (keys %{$cfg->{ids}}) {
6106         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6107     }
6108
6109     print "rescan volumes...\n";
6110     my $volid_hash = scan_volids($cfg, $vmid);
6111
6112     my $updatefn =  sub {
6113         my ($vmid) = @_;
6114
6115         my $conf = PVE::QemuConfig->load_config($vmid);
6116
6117         PVE::QemuConfig->check_lock($conf);
6118
6119         my $vm_volids = {};
6120         foreach my $volid (keys %$volid_hash) {
6121             my $info = $volid_hash->{$volid};
6122             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6123         }
6124
6125         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6126
6127         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6128     };
6129
6130     if (defined($vmid)) {
6131         if ($nolock) {
6132             &$updatefn($vmid);
6133         } else {
6134             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6135         }
6136     } else {
6137         my $vmlist = config_list();
6138         foreach my $vmid (keys %$vmlist) {
6139             if ($nolock) {
6140                 &$updatefn($vmid);
6141             } else {
6142                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6143             }
6144         }
6145     }
6146 }
6147
6148 sub restore_proxmox_backup_archive {
6149     my ($archive, $vmid, $user, $options) = @_;
6150
6151     my $storecfg = PVE::Storage::config();
6152
6153     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6154     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6155
6156     my $fingerprint = $scfg->{fingerprint};
6157     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6158
6159     my $repo = PVE::PBSClient::get_repository($scfg);
6160
6161     # This is only used for `pbs-restore`!
6162     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6163     local $ENV{PBS_PASSWORD} = $password;
6164     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6165
6166     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6167         PVE::Storage::parse_volname($storecfg, $archive);
6168
6169     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6170
6171     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6172
6173     my $tmpdir = "/var/tmp/vzdumptmp$$";
6174     rmtree $tmpdir;
6175     mkpath $tmpdir;
6176
6177     my $conffile = PVE::QemuConfig->config_file($vmid);
6178      # disable interrupts (always do cleanups)
6179     local $SIG{INT} =
6180         local $SIG{TERM} =
6181         local $SIG{QUIT} =
6182         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6183
6184     # Note: $oldconf is undef if VM does not exists
6185     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6186     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6187     my $new_conf_raw = '';
6188
6189     my $rpcenv = PVE::RPCEnvironment::get();
6190     my $devinfo = {};
6191
6192     eval {
6193         # enable interrupts
6194         local $SIG{INT} =
6195             local $SIG{TERM} =
6196             local $SIG{QUIT} =
6197             local $SIG{HUP} =
6198             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6199
6200         my $cfgfn = "$tmpdir/qemu-server.conf";
6201         my $firewall_config_fn = "$tmpdir/fw.conf";
6202         my $index_fn = "$tmpdir/index.json";
6203
6204         my $cmd = "restore";
6205
6206         my $param = [$pbs_backup_name, "index.json", $index_fn];
6207         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6208         my $index = PVE::Tools::file_get_contents($index_fn);
6209         $index = decode_json($index);
6210
6211         # print Dumper($index);
6212         foreach my $info (@{$index->{files}}) {
6213             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6214                 my $devname = $1;
6215                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6216                     $devinfo->{$devname}->{size} = $1;
6217                 } else {
6218                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6219                 }
6220             }
6221         }
6222
6223         my $is_qemu_server_backup = scalar(
6224             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6225         );
6226         if (!$is_qemu_server_backup) {
6227             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6228         }
6229         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6230
6231         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6232         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6233
6234         if ($has_firewall_config) {
6235             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6236             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6237
6238             my $pve_firewall_dir = '/etc/pve/firewall';
6239             mkdir $pve_firewall_dir; # make sure the dir exists
6240             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6241         }
6242
6243         my $fh = IO::File->new($cfgfn, "r") ||
6244             die "unable to read qemu-server.conf - $!\n";
6245
6246         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6247
6248         # fixme: rate limit?
6249
6250         # create empty/temp config
6251         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6252
6253         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6254
6255         # allocate volumes
6256         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6257
6258         foreach my $virtdev (sort keys %$virtdev_hash) {
6259             my $d = $virtdev_hash->{$virtdev};
6260             next if $d->{is_cloudinit}; # no need to restore cloudinit
6261
6262             my $volid = $d->{volid};
6263
6264             my $path = PVE::Storage::path($storecfg, $volid);
6265
6266             # This is the ONLY user of the PBS_ env vars set on top of this function!
6267             my $pbs_restore_cmd = [
6268                 '/usr/bin/pbs-restore',
6269                 '--repository', $repo,
6270                 $pbs_backup_name,
6271                 "$d->{devname}.img.fidx",
6272                 $path,
6273                 '--verbose',
6274                 ];
6275
6276             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6277             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6278
6279             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6280                 push @$pbs_restore_cmd, '--skip-zero';
6281             }
6282
6283             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6284             print "restore proxmox backup image: $dbg_cmdstring\n";
6285             run_command($pbs_restore_cmd);
6286         }
6287
6288         $fh->seek(0, 0) || die "seek failed - $!\n";
6289
6290         my $cookie = { netcount => 0 };
6291         while (defined(my $line = <$fh>)) {
6292             $new_conf_raw .= $restore_update_config_line->(
6293                 $cookie,
6294                 $vmid,
6295                 $map,
6296                 $line,
6297                 $options->{unique},
6298             );
6299         }
6300
6301         $fh->close();
6302     };
6303     my $err = $@;
6304
6305     $restore_deactivate_volumes->($storecfg, $devinfo);
6306
6307     rmtree $tmpdir;
6308
6309     if ($err) {
6310         $restore_destroy_volumes->($storecfg, $devinfo);
6311         die $err;
6312     }
6313
6314     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6315
6316     PVE::Cluster::cfs_update(); # make sure we read new file
6317
6318     eval { rescan($vmid, 1); };
6319     warn $@ if $@;
6320 }
6321
6322 sub restore_vma_archive {
6323     my ($archive, $vmid, $user, $opts, $comp) = @_;
6324
6325     my $readfrom = $archive;
6326
6327     my $cfg = PVE::Storage::config();
6328     my $commands = [];
6329     my $bwlimit = $opts->{bwlimit};
6330
6331     my $dbg_cmdstring = '';
6332     my $add_pipe = sub {
6333         my ($cmd) = @_;
6334         push @$commands, $cmd;
6335         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6336         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6337         $readfrom = '-';
6338     };
6339
6340     my $input = undef;
6341     if ($archive eq '-') {
6342         $input = '<&STDIN';
6343     } else {
6344         # If we use a backup from a PVE defined storage we also consider that
6345         # storage's rate limit:
6346         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6347         if (defined($volid)) {
6348             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6349             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6350             if ($readlimit) {
6351                 print STDERR "applying read rate limit: $readlimit\n";
6352                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6353                 $add_pipe->($cstream);
6354             }
6355         }
6356     }
6357
6358     if ($comp) {
6359         my $info = PVE::Storage::decompressor_info('vma', $comp);
6360         my $cmd = $info->{decompressor};
6361         push @$cmd, $readfrom;
6362         $add_pipe->($cmd);
6363     }
6364
6365     my $tmpdir = "/var/tmp/vzdumptmp$$";
6366     rmtree $tmpdir;
6367
6368     # disable interrupts (always do cleanups)
6369     local $SIG{INT} =
6370         local $SIG{TERM} =
6371         local $SIG{QUIT} =
6372         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6373
6374     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6375     POSIX::mkfifo($mapfifo, 0600);
6376     my $fifofh;
6377     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6378
6379     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6380
6381     my $oldtimeout;
6382     my $timeout = 5;
6383
6384     my $devinfo = {};
6385
6386     my $rpcenv = PVE::RPCEnvironment::get();
6387
6388     my $conffile = PVE::QemuConfig->config_file($vmid);
6389
6390     # Note: $oldconf is undef if VM does not exist
6391     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6392     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6393     my $new_conf_raw = '';
6394
6395     my %storage_limits;
6396
6397     my $print_devmap = sub {
6398         my $cfgfn = "$tmpdir/qemu-server.conf";
6399
6400         # we can read the config - that is already extracted
6401         my $fh = IO::File->new($cfgfn, "r") ||
6402             die "unable to read qemu-server.conf - $!\n";
6403
6404         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6405         if (-f $fwcfgfn) {
6406             my $pve_firewall_dir = '/etc/pve/firewall';
6407             mkdir $pve_firewall_dir; # make sure the dir exists
6408             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6409         }
6410
6411         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6412
6413         foreach my $info (values %{$virtdev_hash}) {
6414             my $storeid = $info->{storeid};
6415             next if defined($storage_limits{$storeid});
6416
6417             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6418             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6419             $storage_limits{$storeid} = $limit * 1024;
6420         }
6421
6422         foreach my $devname (keys %$devinfo) {
6423             die "found no device mapping information for device '$devname'\n"
6424                 if !$devinfo->{$devname}->{virtdev};
6425         }
6426
6427         # create empty/temp config
6428         if ($oldconf) {
6429             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6430             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6431         }
6432
6433         # allocate volumes
6434         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6435
6436         # print restore information to $fifofh
6437         foreach my $virtdev (sort keys %$virtdev_hash) {
6438             my $d = $virtdev_hash->{$virtdev};
6439             next if $d->{is_cloudinit}; # no need to restore cloudinit
6440
6441             my $storeid = $d->{storeid};
6442             my $volid = $d->{volid};
6443
6444             my $map_opts = '';
6445             if (my $limit = $storage_limits{$storeid}) {
6446                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6447             }
6448
6449             my $write_zeros = 1;
6450             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6451                 $write_zeros = 0;
6452             }
6453
6454             my $path = PVE::Storage::path($cfg, $volid);
6455
6456             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6457
6458             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6459         }
6460
6461         $fh->seek(0, 0) || die "seek failed - $!\n";
6462
6463         my $cookie = { netcount => 0 };
6464         while (defined(my $line = <$fh>)) {
6465             $new_conf_raw .= $restore_update_config_line->(
6466                 $cookie,
6467                 $vmid,
6468                 $map,
6469                 $line,
6470                 $opts->{unique},
6471             );
6472         }
6473
6474         $fh->close();
6475     };
6476
6477     eval {
6478         # enable interrupts
6479         local $SIG{INT} =
6480             local $SIG{TERM} =
6481             local $SIG{QUIT} =
6482             local $SIG{HUP} =
6483             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6484         local $SIG{ALRM} = sub { die "got timeout\n"; };
6485
6486         $oldtimeout = alarm($timeout);
6487
6488         my $parser = sub {
6489             my $line = shift;
6490
6491             print "$line\n";
6492
6493             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6494                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6495                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6496             } elsif ($line =~ m/^CTIME: /) {
6497                 # we correctly received the vma config, so we can disable
6498                 # the timeout now for disk allocation (set to 10 minutes, so
6499                 # that we always timeout if something goes wrong)
6500                 alarm(600);
6501                 &$print_devmap();
6502                 print $fifofh "done\n";
6503                 my $tmp = $oldtimeout || 0;
6504                 $oldtimeout = undef;
6505                 alarm($tmp);
6506                 close($fifofh);
6507                 $fifofh = undef;
6508             }
6509         };
6510
6511         print "restore vma archive: $dbg_cmdstring\n";
6512         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6513     };
6514     my $err = $@;
6515
6516     alarm($oldtimeout) if $oldtimeout;
6517
6518     $restore_deactivate_volumes->($cfg, $devinfo);
6519
6520     close($fifofh) if $fifofh;
6521     unlink $mapfifo;
6522     rmtree $tmpdir;
6523
6524     if ($err) {
6525         $restore_destroy_volumes->($cfg, $devinfo);
6526         die $err;
6527     }
6528
6529     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6530
6531     PVE::Cluster::cfs_update(); # make sure we read new file
6532
6533     eval { rescan($vmid, 1); };
6534     warn $@ if $@;
6535 }
6536
6537 sub restore_tar_archive {
6538     my ($archive, $vmid, $user, $opts) = @_;
6539
6540     if ($archive ne '-') {
6541         my $firstfile = tar_archive_read_firstfile($archive);
6542         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6543             if $firstfile ne 'qemu-server.conf';
6544     }
6545
6546     my $storecfg = PVE::Storage::config();
6547
6548     # avoid zombie disks when restoring over an existing VM -> cleanup first
6549     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6550     # skiplock=1 because qmrestore has set the 'create' lock itself already
6551     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6552     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6553
6554     my $tocmd = "/usr/lib/qemu-server/qmextract";
6555
6556     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6557     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6558     $tocmd .= ' --prealloc' if $opts->{prealloc};
6559     $tocmd .= ' --info' if $opts->{info};
6560
6561     # tar option "xf" does not autodetect compression when read from STDIN,
6562     # so we pipe to zcat
6563     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6564         PVE::Tools::shellquote("--to-command=$tocmd");
6565
6566     my $tmpdir = "/var/tmp/vzdumptmp$$";
6567     mkpath $tmpdir;
6568
6569     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6570     local $ENV{VZDUMP_VMID} = $vmid;
6571     local $ENV{VZDUMP_USER} = $user;
6572
6573     my $conffile = PVE::QemuConfig->config_file($vmid);
6574     my $new_conf_raw = '';
6575
6576     # disable interrupts (always do cleanups)
6577     local $SIG{INT} =
6578         local $SIG{TERM} =
6579         local $SIG{QUIT} =
6580         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6581
6582     eval {
6583         # enable interrupts
6584         local $SIG{INT} =
6585             local $SIG{TERM} =
6586             local $SIG{QUIT} =
6587             local $SIG{HUP} =
6588             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6589
6590         if ($archive eq '-') {
6591             print "extracting archive from STDIN\n";
6592             run_command($cmd, input => "<&STDIN");
6593         } else {
6594             print "extracting archive '$archive'\n";
6595             run_command($cmd);
6596         }
6597
6598         return if $opts->{info};
6599
6600         # read new mapping
6601         my $map = {};
6602         my $statfile = "$tmpdir/qmrestore.stat";
6603         if (my $fd = IO::File->new($statfile, "r")) {
6604             while (defined (my $line = <$fd>)) {
6605                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6606                     $map->{$1} = $2 if $1;
6607                 } else {
6608                     print STDERR "unable to parse line in statfile - $line\n";
6609                 }
6610             }
6611             $fd->close();
6612         }
6613
6614         my $confsrc = "$tmpdir/qemu-server.conf";
6615
6616         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6617
6618         my $cookie = { netcount => 0 };
6619         while (defined (my $line = <$srcfd>)) {
6620             $new_conf_raw .= $restore_update_config_line->(
6621                 $cookie,
6622                 $vmid,
6623                 $map,
6624                 $line,
6625                 $opts->{unique},
6626             );
6627         }
6628
6629         $srcfd->close();
6630     };
6631     if (my $err = $@) {
6632         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6633         die $err;
6634     }
6635
6636     rmtree $tmpdir;
6637
6638     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6639
6640     PVE::Cluster::cfs_update(); # make sure we read new file
6641
6642     eval { rescan($vmid, 1); };
6643     warn $@ if $@;
6644 };
6645
6646 sub foreach_storage_used_by_vm {
6647     my ($conf, $func) = @_;
6648
6649     my $sidhash = {};
6650
6651     PVE::QemuConfig->foreach_volume($conf, sub {
6652         my ($ds, $drive) = @_;
6653         return if drive_is_cdrom($drive);
6654
6655         my $volid = $drive->{file};
6656
6657         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6658         $sidhash->{$sid} = $sid if $sid;
6659     });
6660
6661     foreach my $sid (sort keys %$sidhash) {
6662         &$func($sid);
6663     }
6664 }
6665
6666 my $qemu_snap_storage = {
6667     rbd => 1,
6668 };
6669 sub do_snapshots_with_qemu {
6670     my ($storecfg, $volid) = @_;
6671
6672     my $storage_name = PVE::Storage::parse_volume_id($volid);
6673     my $scfg = $storecfg->{ids}->{$storage_name};
6674     die "could not find storage '$storage_name'\n" if !defined($scfg);
6675
6676     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6677         return 1;
6678     }
6679
6680     if ($volid =~ m/\.(qcow2|qed)$/){
6681         return 1;
6682     }
6683
6684     return;
6685 }
6686
6687 sub qga_check_running {
6688     my ($vmid, $nowarn) = @_;
6689
6690     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6691     if ($@) {
6692         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6693         return 0;
6694     }
6695     return 1;
6696 }
6697
6698 sub template_create {
6699     my ($vmid, $conf, $disk) = @_;
6700
6701     my $storecfg = PVE::Storage::config();
6702
6703     PVE::QemuConfig->foreach_volume($conf, sub {
6704         my ($ds, $drive) = @_;
6705
6706         return if drive_is_cdrom($drive);
6707         return if $disk && $ds ne $disk;
6708
6709         my $volid = $drive->{file};
6710         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6711
6712         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6713         $drive->{file} = $voliddst;
6714         $conf->{$ds} = print_drive($drive);
6715         PVE::QemuConfig->write_config($vmid, $conf);
6716     });
6717 }
6718
6719 sub convert_iscsi_path {
6720     my ($path) = @_;
6721
6722     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6723         my $portal = $1;
6724         my $target = $2;
6725         my $lun = $3;
6726
6727         my $initiator_name = get_initiator_name();
6728
6729         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6730                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6731     }
6732
6733     die "cannot convert iscsi path '$path', unkown format\n";
6734 }
6735
6736 sub qemu_img_convert {
6737     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6738
6739     my $storecfg = PVE::Storage::config();
6740     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6741     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6742
6743     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6744
6745     my $cachemode;
6746     my $src_path;
6747     my $src_is_iscsi = 0;
6748     my $src_format;
6749
6750     if ($src_storeid) {
6751         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6752         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6753         $src_format = qemu_img_format($src_scfg, $src_volname);
6754         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6755         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6756         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6757     } elsif (-f $src_volid) {
6758         $src_path = $src_volid;
6759         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6760             $src_format = $1;
6761         }
6762     }
6763
6764     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6765
6766     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6767     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6768     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6769     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6770
6771     my $cmd = [];
6772     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6773     push @$cmd, '-l', "snapshot.name=$snapname"
6774         if $snapname && $src_format && $src_format eq "qcow2";
6775     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6776     push @$cmd, '-T', $cachemode if defined($cachemode);
6777
6778     if ($src_is_iscsi) {
6779         push @$cmd, '--image-opts';
6780         $src_path = convert_iscsi_path($src_path);
6781     } elsif ($src_format) {
6782         push @$cmd, '-f', $src_format;
6783     }
6784
6785     if ($dst_is_iscsi) {
6786         push @$cmd, '--target-image-opts';
6787         $dst_path = convert_iscsi_path($dst_path);
6788     } else {
6789         push @$cmd, '-O', $dst_format;
6790     }
6791
6792     push @$cmd, $src_path;
6793
6794     if (!$dst_is_iscsi && $is_zero_initialized) {
6795         push @$cmd, "zeroinit:$dst_path";
6796     } else {
6797         push @$cmd, $dst_path;
6798     }
6799
6800     my $parser = sub {
6801         my $line = shift;
6802         if($line =~ m/\((\S+)\/100\%\)/){
6803             my $percent = $1;
6804             my $transferred = int($size * $percent / 100);
6805             my $remaining = $size - $transferred;
6806
6807             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6808         }
6809
6810     };
6811
6812     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6813     my $err = $@;
6814     die "copy failed: $err" if $err;
6815 }
6816
6817 sub qemu_img_format {
6818     my ($scfg, $volname) = @_;
6819
6820     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6821         return $1;
6822     } else {
6823         return "raw";
6824     }
6825 }
6826
6827 sub qemu_drive_mirror {
6828     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6829
6830     $jobs = {} if !$jobs;
6831
6832     my $qemu_target;
6833     my $format;
6834     $jobs->{"drive-$drive"} = {};
6835
6836     if ($dst_volid =~ /^nbd:/) {
6837         $qemu_target = $dst_volid;
6838         $format = "nbd";
6839     } else {
6840         my $storecfg = PVE::Storage::config();
6841         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6842
6843         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6844
6845         $format = qemu_img_format($dst_scfg, $dst_volname);
6846
6847         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6848
6849         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6850     }
6851
6852     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6853     $opts->{format} = $format if $format;
6854
6855     if (defined($src_bitmap)) {
6856         $opts->{sync} = 'incremental';
6857         $opts->{bitmap} = $src_bitmap;
6858         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6859     }
6860
6861     if (defined($bwlimit)) {
6862         $opts->{speed} = $bwlimit * 1024;
6863         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6864     } else {
6865         print "drive mirror is starting for drive-$drive\n";
6866     }
6867
6868     # if a job already runs for this device we get an error, catch it for cleanup
6869     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6870     if (my $err = $@) {
6871         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6872         warn "$@\n" if $@;
6873         die "mirroring error: $err\n";
6874     }
6875
6876     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6877 }
6878
6879 # $completion can be either
6880 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6881 # 'cancel': wait until all jobs are ready, block-job-cancel them
6882 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6883 sub qemu_drive_mirror_monitor {
6884     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6885
6886     $completion //= 'complete';
6887
6888     eval {
6889         my $err_complete = 0;
6890
6891         while (1) {
6892             die "storage migration timed out\n" if $err_complete > 300;
6893
6894             my $stats = mon_cmd($vmid, "query-block-jobs");
6895
6896             my $running_mirror_jobs = {};
6897             foreach my $stat (@$stats) {
6898                 next if $stat->{type} ne 'mirror';
6899                 $running_mirror_jobs->{$stat->{device}} = $stat;
6900             }
6901
6902             my $readycounter = 0;
6903
6904             foreach my $job (keys %$jobs) {
6905
6906                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6907                     print "$job : finished\n";
6908                     delete $jobs->{$job};
6909                     next;
6910                 }
6911
6912                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6913
6914                 my $busy = $running_mirror_jobs->{$job}->{busy};
6915                 my $ready = $running_mirror_jobs->{$job}->{ready};
6916                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6917                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6918                     my $remaining = $total - $transferred;
6919                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6920
6921                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6922                 }
6923
6924                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6925             }
6926
6927             last if scalar(keys %$jobs) == 0;
6928
6929             if ($readycounter == scalar(keys %$jobs)) {
6930                 print "all mirroring jobs are ready \n";
6931                 last if $completion eq 'skip'; #do the complete later
6932
6933                 if ($vmiddst && $vmiddst != $vmid) {
6934                     my $agent_running = $qga && qga_check_running($vmid);
6935                     if ($agent_running) {
6936                         print "freeze filesystem\n";
6937                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6938                     } else {
6939                         print "suspend vm\n";
6940                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6941                     }
6942
6943                     # if we clone a disk for a new target vm, we don't switch the disk
6944                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6945
6946                     if ($agent_running) {
6947                         print "unfreeze filesystem\n";
6948                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6949                     } else {
6950                         print "resume vm\n";
6951                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6952                     }
6953
6954                     last;
6955                 } else {
6956
6957                     foreach my $job (keys %$jobs) {
6958                         # try to switch the disk if source and destination are on the same guest
6959                         print "$job: Completing block job...\n";
6960
6961                         my $op;
6962                         if ($completion eq 'complete') {
6963                             $op = 'block-job-complete';
6964                         } elsif ($completion eq 'cancel') {
6965                             $op = 'block-job-cancel';
6966                         } else {
6967                             die "invalid completion value: $completion\n";
6968                         }
6969                         eval { mon_cmd($vmid, $op, device => $job) };
6970                         if ($@ =~ m/cannot be completed/) {
6971                             print "$job: Block job cannot be completed, try again.\n";
6972                             $err_complete++;
6973                         }else {
6974                             print "$job: Completed successfully.\n";
6975                             $jobs->{$job}->{complete} = 1;
6976                         }
6977                     }
6978                 }
6979             }
6980             sleep 1;
6981         }
6982     };
6983     my $err = $@;
6984
6985     if ($err) {
6986         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6987         die "mirroring error: $err";
6988     }
6989
6990 }
6991
6992 sub qemu_blockjobs_cancel {
6993     my ($vmid, $jobs) = @_;
6994
6995     foreach my $job (keys %$jobs) {
6996         print "$job: Cancelling block job\n";
6997         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6998         $jobs->{$job}->{cancel} = 1;
6999     }
7000
7001     while (1) {
7002         my $stats = mon_cmd($vmid, "query-block-jobs");
7003
7004         my $running_jobs = {};
7005         foreach my $stat (@$stats) {
7006             $running_jobs->{$stat->{device}} = $stat;
7007         }
7008
7009         foreach my $job (keys %$jobs) {
7010
7011             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7012                 print "$job: Done.\n";
7013                 delete $jobs->{$job};
7014             }
7015         }
7016
7017         last if scalar(keys %$jobs) == 0;
7018
7019         sleep 1;
7020     }
7021 }
7022
7023 sub clone_disk {
7024     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7025         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7026
7027     my $newvolid;
7028
7029     if (!$full) {
7030         print "create linked clone of drive $drivename ($drive->{file})\n";
7031         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7032         push @$newvollist, $newvolid;
7033     } else {
7034
7035         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7036         $storeid = $storage if $storage;
7037
7038         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7039
7040         print "create full clone of drive $drivename ($drive->{file})\n";
7041         my $name = undef;
7042         my $size = undef;
7043         if (drive_is_cloudinit($drive)) {
7044             $name = "vm-$newvmid-cloudinit";
7045             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7046             if ($scfg->{path}) {
7047                 $name .= ".$dst_format";
7048             }
7049             $snapname = undef;
7050             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7051         } elsif ($drivename eq 'efidisk0') {
7052             $size = get_efivars_size($conf);
7053         } else {
7054             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7055         }
7056         $newvolid = PVE::Storage::vdisk_alloc(
7057             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7058         );
7059         push @$newvollist, $newvolid;
7060
7061         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7062
7063         if (drive_is_cloudinit($drive)) {
7064             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7065             # if this is the case, we have to complete any block-jobs still there from
7066             # previous drive-mirrors
7067             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7068                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7069             }
7070             goto no_data_clone;
7071         }
7072
7073         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7074         if (!$running || $snapname) {
7075             # TODO: handle bwlimits
7076             if ($drivename eq 'efidisk0') {
7077                 # the relevant data on the efidisk may be smaller than the source
7078                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7079                 # that is given by the OVMF_VARS.fd
7080                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7081                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7082
7083                 # better for Ceph if block size is not too small, see bug #3324
7084                 my $bs = 1024*1024;
7085
7086                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7087                     "if=$src_path", "of=$dst_path"]);
7088             } else {
7089                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7090             }
7091         } else {
7092
7093             my $kvmver = get_running_qemu_version ($vmid);
7094             if (!min_version($kvmver, 2, 7)) {
7095                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7096                     if $drive->{iothread};
7097             }
7098
7099             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7100                 $completion, $qga, $bwlimit);
7101         }
7102     }
7103
7104 no_data_clone:
7105     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7106
7107     my $disk = $drive;
7108     $disk->{format} = undef;
7109     $disk->{file} = $newvolid;
7110     $disk->{size} = $size if defined($size);
7111
7112     return $disk;
7113 }
7114
7115 sub get_running_qemu_version {
7116     my ($vmid) = @_;
7117     my $res = mon_cmd($vmid, "query-version");
7118     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7119 }
7120
7121 sub qemu_use_old_bios_files {
7122     my ($machine_type) = @_;
7123
7124     return if !$machine_type;
7125
7126     my $use_old_bios_files = undef;
7127
7128     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7129         $machine_type = $1;
7130         $use_old_bios_files = 1;
7131     } else {
7132         my $version = extract_version($machine_type, kvm_user_version());
7133         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7134         # load new efi bios files on migration. So this hack is required to allow
7135         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7136         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7137         $use_old_bios_files = !min_version($version, 2, 4);
7138     }
7139
7140     return ($use_old_bios_files, $machine_type);
7141 }
7142
7143 sub get_efivars_size {
7144     my ($conf) = @_;
7145     my $arch = get_vm_arch($conf);
7146     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7147     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7148     return -s $ovmf_vars;
7149 }
7150
7151 sub update_efidisk_size {
7152     my ($conf) = @_;
7153
7154     return if !defined($conf->{efidisk0});
7155
7156     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7157     $disk->{size} = get_efivars_size($conf);
7158     $conf->{efidisk0} = print_drive($disk);
7159
7160     return;
7161 }
7162
7163 sub create_efidisk($$$$$) {
7164     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7165
7166     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7167     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7168
7169     my $vars_size_b = -s $ovmf_vars;
7170     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7171     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7172     PVE::Storage::activate_volumes($storecfg, [$volid]);
7173
7174     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7175     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7176
7177     return ($volid, $size/1024);
7178 }
7179
7180 sub vm_iothreads_list {
7181     my ($vmid) = @_;
7182
7183     my $res = mon_cmd($vmid, 'query-iothreads');
7184
7185     my $iothreads = {};
7186     foreach my $iothread (@$res) {
7187         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7188     }
7189
7190     return $iothreads;
7191 }
7192
7193 sub scsihw_infos {
7194     my ($conf, $drive) = @_;
7195
7196     my $maxdev = 0;
7197
7198     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7199         $maxdev = 7;
7200     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7201         $maxdev = 1;
7202     } else {
7203         $maxdev = 256;
7204     }
7205
7206     my $controller = int($drive->{index} / $maxdev);
7207     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7208         ? "virtioscsi"
7209         : "scsihw";
7210
7211     return ($maxdev, $controller, $controller_prefix);
7212 }
7213
7214 sub windows_version {
7215     my ($ostype) = @_;
7216
7217     return 0 if !$ostype;
7218
7219     my $winversion = 0;
7220
7221     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7222         $winversion = 5;
7223     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7224         $winversion = 6;
7225     } elsif ($ostype =~ m/^win(\d+)$/) {
7226         $winversion = $1;
7227     }
7228
7229     return $winversion;
7230 }
7231
7232 sub resolve_dst_disk_format {
7233         my ($storecfg, $storeid, $src_volname, $format) = @_;
7234         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7235
7236         if (!$format) {
7237             # if no target format is specified, use the source disk format as hint
7238             if ($src_volname) {
7239                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7240                 $format = qemu_img_format($scfg, $src_volname);
7241             } else {
7242                 return $defFormat;
7243             }
7244         }
7245
7246         # test if requested format is supported - else use default
7247         my $supported = grep { $_ eq $format } @$validFormats;
7248         $format = $defFormat if !$supported;
7249         return $format;
7250 }
7251
7252 # NOTE: if this logic changes, please update docs & possibly gui logic
7253 sub find_vmstate_storage {
7254     my ($conf, $storecfg) = @_;
7255
7256     # first, return storage from conf if set
7257     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7258
7259     my ($target, $shared, $local);
7260
7261     foreach_storage_used_by_vm($conf, sub {
7262         my ($sid) = @_;
7263         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7264         my $dst = $scfg->{shared} ? \$shared : \$local;
7265         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7266     });
7267
7268     # second, use shared storage where VM has at least one disk
7269     # third, use local storage where VM has at least one disk
7270     # fall back to local storage
7271     $target = $shared // $local // 'local';
7272
7273     return $target;
7274 }
7275
7276 sub generate_uuid {
7277     my ($uuid, $uuid_str);
7278     UUID::generate($uuid);
7279     UUID::unparse($uuid, $uuid_str);
7280     return $uuid_str;
7281 }
7282
7283 sub generate_smbios1_uuid {
7284     return "uuid=".generate_uuid();
7285 }
7286
7287 sub nbd_stop {
7288     my ($vmid) = @_;
7289
7290     mon_cmd($vmid, 'nbd-server-stop');
7291 }
7292
7293 sub create_reboot_request {
7294     my ($vmid) = @_;
7295     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7296         or die "failed to create reboot trigger file: $!\n";
7297     close($fh);
7298 }
7299
7300 sub clear_reboot_request {
7301     my ($vmid) = @_;
7302     my $path = "/run/qemu-server/$vmid.reboot";
7303     my $res = 0;
7304
7305     $res = unlink($path);
7306     die "could not remove reboot request for $vmid: $!"
7307         if !$res && $! != POSIX::ENOENT;
7308
7309     return $res;
7310 }
7311
7312 sub bootorder_from_legacy {
7313     my ($conf, $bootcfg) = @_;
7314
7315     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7316     my $bootindex_hash = {};
7317     my $i = 1;
7318     foreach my $o (split(//, $boot)) {
7319         $bootindex_hash->{$o} = $i*100;
7320         $i++;
7321     }
7322
7323     my $bootorder = {};
7324
7325     PVE::QemuConfig->foreach_volume($conf, sub {
7326         my ($ds, $drive) = @_;
7327
7328         if (drive_is_cdrom ($drive, 1)) {
7329             if ($bootindex_hash->{d}) {
7330                 $bootorder->{$ds} = $bootindex_hash->{d};
7331                 $bootindex_hash->{d} += 1;
7332             }
7333         } elsif ($bootindex_hash->{c}) {
7334             $bootorder->{$ds} = $bootindex_hash->{c}
7335                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7336             $bootindex_hash->{c} += 1;
7337         }
7338     });
7339
7340     if ($bootindex_hash->{n}) {
7341         for (my $i = 0; $i < $MAX_NETS; $i++) {
7342             my $netname = "net$i";
7343             next if !$conf->{$netname};
7344             $bootorder->{$netname} = $bootindex_hash->{n};
7345             $bootindex_hash->{n} += 1;
7346         }
7347     }
7348
7349     return $bootorder;
7350 }
7351
7352 # Generate default device list for 'boot: order=' property. Matches legacy
7353 # default boot order, but with explicit device names. This is important, since
7354 # the fallback for when neither 'order' nor the old format is specified relies
7355 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7356 sub get_default_bootdevices {
7357     my ($conf) = @_;
7358
7359     my @ret = ();
7360
7361     # harddisk
7362     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7363     push @ret, $first if $first;
7364
7365     # cdrom
7366     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7367     push @ret, $first if $first;
7368
7369     # network
7370     for (my $i = 0; $i < $MAX_NETS; $i++) {
7371         my $netname = "net$i";
7372         next if !$conf->{$netname};
7373         push @ret, $netname;
7374         last;
7375     }
7376
7377     return \@ret;
7378 }
7379
7380 sub device_bootorder {
7381     my ($conf) = @_;
7382
7383     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7384
7385     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7386
7387     my $bootorder = {};
7388     if (!defined($boot) || $boot->{legacy}) {
7389         $bootorder = bootorder_from_legacy($conf, $boot);
7390     } elsif ($boot->{order}) {
7391         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7392         for my $dev (PVE::Tools::split_list($boot->{order})) {
7393             $bootorder->{$dev} = $i++;
7394         }
7395     }
7396
7397     return $bootorder;
7398 }
7399
7400 # bash completion helper
7401
7402 sub complete_backup_archives {
7403     my ($cmdname, $pname, $cvalue) = @_;
7404
7405     my $cfg = PVE::Storage::config();
7406
7407     my $storeid;
7408
7409     if ($cvalue =~ m/^([^:]+):/) {
7410         $storeid = $1;
7411     }
7412
7413     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7414
7415     my $res = [];
7416     foreach my $id (keys %$data) {
7417         foreach my $item (@{$data->{$id}}) {
7418             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7419             push @$res, $item->{volid} if defined($item->{volid});
7420         }
7421     }
7422
7423     return $res;
7424 }
7425
7426 my $complete_vmid_full = sub {
7427     my ($running) = @_;
7428
7429     my $idlist = vmstatus();
7430
7431     my $res = [];
7432
7433     foreach my $id (keys %$idlist) {
7434         my $d = $idlist->{$id};
7435         if (defined($running)) {
7436             next if $d->{template};
7437             next if $running && $d->{status} ne 'running';
7438             next if !$running && $d->{status} eq 'running';
7439         }
7440         push @$res, $id;
7441
7442     }
7443     return $res;
7444 };
7445
7446 sub complete_vmid {
7447     return &$complete_vmid_full();
7448 }
7449
7450 sub complete_vmid_stopped {
7451     return &$complete_vmid_full(0);
7452 }
7453
7454 sub complete_vmid_running {
7455     return &$complete_vmid_full(1);
7456 }
7457
7458 sub complete_storage {
7459
7460     my $cfg = PVE::Storage::config();
7461     my $ids = $cfg->{ids};
7462
7463     my $res = [];
7464     foreach my $sid (keys %$ids) {
7465         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7466         next if !$ids->{$sid}->{content}->{images};
7467         push @$res, $sid;
7468     }
7469
7470     return $res;
7471 }
7472
7473 sub complete_migration_storage {
7474     my ($cmd, $param, $current_value, $all_args) = @_;
7475
7476     my $targetnode = @$all_args[1];
7477
7478     my $cfg = PVE::Storage::config();
7479     my $ids = $cfg->{ids};
7480
7481     my $res = [];
7482     foreach my $sid (keys %$ids) {
7483         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7484         next if !$ids->{$sid}->{content}->{images};
7485         push @$res, $sid;
7486     }
7487
7488     return $res;
7489 }
7490
7491 sub vm_is_paused {
7492     my ($vmid) = @_;
7493     my $qmpstatus = eval {
7494         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7495         mon_cmd($vmid, "query-status");
7496     };
7497     warn "$@\n" if $@;
7498     return $qmpstatus && $qmpstatus->{status} eq "paused";
7499 }
7500
7501 1;