PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Shown in the web-interface VM's summary."
 369             ." This is saved as comment inside the configuration file.",
 370         maxLength => 1024 * 8,
 371     },
 372     ostype => {
 373         optional => 1,
 374         type => 'string',
 375         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 376         description => "Specify guest operating system.",
 377         verbose_description => <<EODESC,
 378 Specify guest operating system. This is used to enable special
 379 optimization/features for specific operating systems:
 380
 381 [horizontal]
 382 other;; unspecified OS
 383 wxp;; Microsoft Windows XP
 384 w2k;; Microsoft Windows 2000
 385 w2k3;; Microsoft Windows 2003
 386 w2k8;; Microsoft Windows 2008
 387 wvista;; Microsoft Windows Vista
 388 win7;; Microsoft Windows 7
 389 win8;; Microsoft Windows 8/2012/2012r2
 390 win10;; Microsoft Windows 10/2016/2019
 391 l24;; Linux 2.4 Kernel
 392 l26;; Linux 2.6 - 5.X Kernel
 393 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 394 EODESC
 395     },
 396     boot => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-boot',
 399         description => "Specify guest boot order. Use with 'order=', usage with"
 400                      . " no key or 'legacy=' is deprecated.",
 401     },
 402     bootdisk => {
 403         optional => 1,
 404         type => 'string', format => 'pve-qm-bootdisk',
 405         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 406         pattern => '(ide|sata|scsi|virtio)\d+',
 407     },
 408     smp => {
 409         optional => 1,
 410         type => 'integer',
 411         description => "The number of CPUs. Please use option -sockets instead.",
 412         minimum => 1,
 413         default => 1,
 414     },
 415     sockets => {
 416         optional => 1,
 417         type => 'integer',
 418         description => "The number of CPU sockets.",
 419         minimum => 1,
 420         default => 1,
 421     },
 422     cores => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "The number of cores per socket.",
 426         minimum => 1,
 427         default => 1,
 428     },
 429     numa => {
 430         optional => 1,
 431         type => 'boolean',
 432         description => "Enable/disable NUMA.",
 433         default => 0,
 434     },
 435     hugepages => {
 436         optional => 1,
 437         type => 'string',
 438         description => "Enable/disable hugepages memory.",
 439         enum => [qw(any 2 1024)],
 440     },
 441     keephugepages => {
 442         optional => 1,
 443         type => 'boolean',
 444         default => 0,
 445         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 446             ." after VM shutdown and can be used for subsequent starts.",
 447     },
 448     vcpus => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "Number of hotplugged vcpus.",
 452         minimum => 1,
 453         default => 0,
 454     },
 455     acpi => {
 456         optional => 1,
 457         type => 'boolean',
 458         description => "Enable/disable ACPI.",
 459         default => 1,
 460     },
 461     agent => {
 462         optional => 1,
 463         description => "Enable/disable Qemu GuestAgent and its properties.",
 464         type => 'string',
 465         format => $agent_fmt,
 466     },
 467     kvm => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable KVM hardware virtualization.",
 471         default => 1,
 472     },
 473     tdf => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable time drift fix.",
 477         default => 0,
 478     },
 479     localtime => {
 480         optional => 1,
 481         type => 'boolean',
 482         description => "Set the real time clock to local time. This is enabled by default if ostype"
 483             ." indicates a Microsoft OS.",
 484     },
 485     freeze => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 489     },
 490     vga => {
 491         optional => 1,
 492         type => 'string', format => $vga_fmt,
 493         description => "Configure the VGA hardware.",
 494         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 495             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 496             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 497             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 498             ." display server. For win* OS you can select how many independent displays you want,"
 499             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 500             ." using a serial device as terminal.",
 501     },
 502     watchdog => {
 503         optional => 1,
 504         type => 'string', format => 'pve-qm-watchdog',
 505         description => "Create a virtual hardware watchdog device.",
 506         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 507             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 508             ." else the watchdog will reset the guest (or execute the respective action specified)",
 509     },
 510     startdate => {
 511         optional => 1,
 512         type => 'string',
 513         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 514         description => "Set the initial date of the real time clock. Valid format for date are:"
 515             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 516         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 517         default => 'now',
 518     },
 519     startup =>  get_standard_option('pve-startup-order'),
 520     template => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Enable/disable Template.",
 524         default => 0,
 525     },
 526     args => {
 527         optional => 1,
 528         type => 'string',
 529         description => "Arbitrary arguments passed to kvm.",
 530         verbose_description => <<EODESCR,
 531 Arbitrary arguments passed to kvm, for example:
 532
 533 args: -no-reboot -no-hpet
 534
 535 NOTE: this option is for experts only.
 536 EODESCR
 537     },
 538     tablet => {
 539         optional => 1,
 540         type => 'boolean',
 541         default => 1,
 542         description => "Enable/disable the USB tablet device.",
 543         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 544             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 545             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 546             ." may consider disabling this to save some context switches. This is turned off by"
 547             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 548     },
 549     migrate_speed => {
 550         optional => 1,
 551         type => 'integer',
 552         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 553         minimum => 0,
 554         default => 0,
 555     },
 556     migrate_downtime => {
 557         optional => 1,
 558         type => 'number',
 559         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 560         minimum => 0,
 561         default => 0.1,
 562     },
 563     cdrom => {
 564         optional => 1,
 565         type => 'string', format => 'pve-qm-ide',
 566         typetext => '<volume>',
 567         description => "This is an alias for option -ide2",
 568     },
 569     cpu => {
 570         optional => 1,
 571         description => "Emulated CPU type.",
 572         type => 'string',
 573         format => 'pve-vm-cpu-conf',
 574     },
 575     parent => get_standard_option('pve-snapshot-name', {
 576         optional => 1,
 577         description => "Parent snapshot name. This is used internally, and should not be modified.",
 578     }),
 579     snaptime => {
 580         optional => 1,
 581         description => "Timestamp for snapshots.",
 582         type => 'integer',
 583         minimum => 0,
 584     },
 585     vmstate => {
 586         optional => 1,
 587         type => 'string', format => 'pve-volume-id',
 588         description => "Reference to a volume which stores the VM state. This is used internally"
 589             ." for snapshots.",
 590     },
 591     vmstatestorage => get_standard_option('pve-storage-id', {
 592         description => "Default storage for VM state volumes/files.",
 593         optional => 1,
 594     }),
 595     runningmachine => get_standard_option('pve-qemu-machine', {
 596         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 597             ." for snapshots.",
 598     }),
 599     runningcpu => {
 600         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 601             ." internally for snapshots.",
 602         optional => 1,
 603         type => 'string',
 604         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 605         format_description => 'QEMU -cpu parameter'
 606     },
 607     machine => get_standard_option('pve-qemu-machine'),
 608     arch => {
 609         description => "Virtual processor architecture. Defaults to the host.",
 610         optional => 1,
 611         type => 'string',
 612         enum => [qw(x86_64 aarch64)],
 613     },
 614     smbios1 => {
 615         description => "Specify SMBIOS type 1 fields.",
 616         type => 'string', format => 'pve-qm-smbios1',
 617         maxLength => 512,
 618         optional => 1,
 619     },
 620     protection => {
 621         optional => 1,
 622         type => 'boolean',
 623         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 624             ." remove disk operations.",
 625         default => 0,
 626     },
 627     bios => {
 628         optional => 1,
 629         type => 'string',
 630         enum => [ qw(seabios ovmf) ],
 631         description => "Select BIOS implementation.",
 632         default => 'seabios',
 633     },
 634     vmgenid => {
 635         type => 'string',
 636         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 637         format_description => 'UUID',
 638         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 639             ." to disable explicitly.",
 640         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 641             ." value identifier to the guest OS. This allows to notify the guest operating system"
 642             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 643             ." execution or creation from a template). The guest operating system notices the"
 644             ." change, and is then able to react as appropriate by marking its copies of"
 645             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 646             ."Note that auto-creation only works when done through API/CLI create or update methods"
 647             .", but not when manually editing the config file.",
 648         default => "1 (autogenerated)",
 649         optional => 1,
 650     },
 651     hookscript => {
 652         type => 'string',
 653         format => 'pve-volume-id',
 654         optional => 1,
 655         description => "Script that will be executed during various steps in the vms lifetime.",
 656     },
 657     ivshmem => {
 658         type => 'string',
 659         format => $ivshmem_fmt,
 660         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 661             ." the host.",
 662         optional => 1,
 663     },
 664     audio0 => {
 665         type => 'string',
 666         format => $audio_fmt,
 667         description => "Configure a audio device, useful in combination with QXL/Spice.",
 668         optional => 1
 669     },
 670     spice_enhancements => {
 671         type => 'string',
 672         format => $spice_enhancements_fmt,
 673         description => "Configure additional enhancements for SPICE.",
 674         optional => 1
 675     },
 676     tags => {
 677         type => 'string', format => 'pve-tag-list',
 678         description => 'Tags of the VM. This is only meta information.',
 679         optional => 1,
 680     },
 681     rng0 => {
 682         type => 'string',
 683         format => $rng_fmt,
 684         description => "Configure a VirtIO-based Random Number Generator.",
 685         optional => 1,
 686     },
 687 };
 688
 689 my $cicustom_fmt = {
 690     meta => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all meta data passed to the VM via"
 694             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 695         format => 'pve-volume-id',
 696         format_description => 'volume',
 697     },
 698     network => {
 699         type => 'string',
 700         optional => 1,
 701         description => 'Specify a custom file containing all network data passed to the VM via'
 702             .' cloud-init.',
 703         format => 'pve-volume-id',
 704         format_description => 'volume',
 705     },
 706     user => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all user data passed to the VM via'
 710             .' cloud-init.',
 711         format => 'pve-volume-id',
 712         format_description => 'volume',
 713     },
 714 };
 715 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 716
 717 my $confdesc_cloudinit = {
 718     citype => {
 719         optional => 1,
 720         type => 'string',
 721         description => 'Specifies the cloud-init configuration format. The default depends on the'
 722             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 723             .' and `configdrive2` for windows.',
 724         enum => ['configdrive2', 'nocloud', 'opennebula'],
 725     },
 726     ciuser => {
 727         optional => 1,
 728         type => 'string',
 729         description => "cloud-init: User name to change ssh keys and password for instead of the"
 730             ." image's configured default user.",
 731     },
 732     cipassword => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'cloud-init: Password to assign the user. Using this is generally not'
 736             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 737             .' support hashed passwords.',
 738     },
 739     cicustom => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'cloud-init: Specify custom files to replace the automatically generated'
 743             .' ones at start.',
 744         format => 'pve-qm-cicustom',
 745     },
 746     searchdomain => {
 747         optional => 1,
 748         type => 'string',
 749         description => "cloud-init: Sets DNS search domains for a container. Create will'
 750             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 751             .' are set.",
 752     },
 753     nameserver => {
 754         optional => 1,
 755         type => 'string', format => 'address-list',
 756         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 757             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 758             .' are set.",
 759     },
 760     sshkeys => {
 761         optional => 1,
 762         type => 'string',
 763         format => 'urlencoded',
 764         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 765     },
 766 };
 767
 768 # what about other qemu settings ?
 769 #cpu => 'string',
 770 #machine => 'string',
 771 #fda => 'file',
 772 #fdb => 'file',
 773 #mtdblock => 'file',
 774 #sd => 'file',
 775 #pflash => 'file',
 776 #snapshot => 'bool',
 777 #bootp => 'file',
 778 ##tftp => 'dir',
 779 ##smb => 'dir',
 780 #kernel => 'file',
 781 #append => 'string',
 782 #initrd => 'file',
 783 ##soundhw => 'string',
 784
 785 while (my ($k, $v) = each %$confdesc) {
 786     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 787 }
 788
 789 my $MAX_USB_DEVICES = 5;
 790 my $MAX_NETS = 32;
 791 my $MAX_SERIAL_PORTS = 4;
 792 my $MAX_PARALLEL_PORTS = 3;
 793 my $MAX_NUMA = 8;
 794
 795 my $numa_fmt = {
 796     cpus => {
 797         type => "string",
 798         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 799         description => "CPUs accessing this NUMA node.",
 800         format_description => "id[-id];...",
 801     },
 802     memory => {
 803         type => "number",
 804         description => "Amount of memory this NUMA node provides.",
 805         optional => 1,
 806     },
 807     hostnodes => {
 808         type => "string",
 809         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 810         description => "Host NUMA nodes to use.",
 811         format_description => "id[-id];...",
 812         optional => 1,
 813     },
 814     policy => {
 815         type => 'string',
 816         enum => [qw(preferred bind interleave)],
 817         description => "NUMA allocation policy.",
 818         optional => 1,
 819     },
 820 };
 821 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 822 my $numadesc = {
 823     optional => 1,
 824     type => 'string', format => $numa_fmt,
 825     description => "NUMA topology.",
 826 };
 827 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 828
 829 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 830     $confdesc->{"numa$i"} = $numadesc;
 831 }
 832
 833 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 834                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 835                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 836 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 837
 838 my $net_fmt_bridge_descr = <<__EOD__;
 839 Bridge to attach the network device to. The Proxmox VE standard bridge
 840 is called 'vmbr0'.
 841
 842 If you do not specify a bridge, we create a kvm user (NATed) network
 843 device, which provides DHCP and DNS services. The following addresses
 844 are used:
 845
 846  10.0.2.2   Gateway
 847  10.0.2.3   DNS Server
 848  10.0.2.4   SMB Server
 849
 850 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 851 __EOD__
 852
 853 my $net_fmt = {
 854     macaddr  => get_standard_option('mac-addr', {
 855         description => "MAC address. That address must be unique withing your network. This is"
 856             ." automatically generated if not specified.",
 857     }),
 858     model => {
 859         type => 'string',
 860         description => "Network Card Model. The 'virtio' model provides the best performance with"
 861             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 862             ." best to use 'e1000'.",
 863         enum => $nic_model_list,
 864         default_key => 1,
 865     },
 866     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 867     bridge => {
 868         type => 'string',
 869         description => $net_fmt_bridge_descr,
 870         format_description => 'bridge',
 871         pattern => '[-_.\w\d]+',
 872         optional => 1,
 873     },
 874     queues => {
 875         type => 'integer',
 876         minimum => 0, maximum => 16,
 877         description => 'Number of packet queues to be used on the device.',
 878         optional => 1,
 879     },
 880     rate => {
 881         type => 'number',
 882         minimum => 0,
 883         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 884         optional => 1,
 885     },
 886     tag => {
 887         type => 'integer',
 888         minimum => 1, maximum => 4094,
 889         description => 'VLAN tag to apply to packets on this interface.',
 890         optional => 1,
 891     },
 892     trunks => {
 893         type => 'string',
 894         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 895         description => 'VLAN trunks to pass through this interface.',
 896         format_description => 'vlanid[;vlanid...]',
 897         optional => 1,
 898     },
 899     firewall => {
 900         type => 'boolean',
 901         description => 'Whether this interface should be protected by the firewall.',
 902         optional => 1,
 903     },
 904     link_down => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be disconnected (like pulling the plug).',
 907         optional => 1,
 908     },
 909     mtu => {
 910         type => 'integer',
 911         minimum => 1, maximum => 65520,
 912         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 913         optional => 1,
 914     },
 915 };
 916
 917 my $netdesc = {
 918     optional => 1,
 919     type => 'string', format => $net_fmt,
 920     description => "Specify network devices.",
 921 };
 922
 923 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 924
 925 my $ipconfig_fmt = {
 926     ip => {
 927         type => 'string',
 928         format => 'pve-ipv4-config',
 929         format_description => 'IPv4Format/CIDR',
 930         description => 'IPv4 address in CIDR format.',
 931         optional => 1,
 932         default => 'dhcp',
 933     },
 934     gw => {
 935         type => 'string',
 936         format => 'ipv4',
 937         format_description => 'GatewayIPv4',
 938         description => 'Default gateway for IPv4 traffic.',
 939         optional => 1,
 940         requires => 'ip',
 941     },
 942     ip6 => {
 943         type => 'string',
 944         format => 'pve-ipv6-config',
 945         format_description => 'IPv6Format/CIDR',
 946         description => 'IPv6 address in CIDR format.',
 947         optional => 1,
 948         default => 'dhcp',
 949     },
 950     gw6 => {
 951         type => 'string',
 952         format => 'ipv6',
 953         format_description => 'GatewayIPv6',
 954         description => 'Default gateway for IPv6 traffic.',
 955         optional => 1,
 956         requires => 'ip6',
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 960 my $ipconfigdesc = {
 961     optional => 1,
 962     type => 'string', format => 'pve-qm-ipconfig',
 963     description => <<'EODESCR',
 964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 965
 966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 967
 968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 969 gateway should be provided.
 970 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 971 cloud-init 19.4 or newer.
 972
 973 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 974 dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $usb_fmt = {
1006     host => {
1007         default_key => 1,
1008         type => 'string', format => 'pve-qm-usb-device',
1009         format_description => 'HOSTUSBDEVICE|spice',
1010         description => <<EODESCR,
1011 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1012
1013  'bus-port(.port)*' (decimal numbers) or
1014  'vendor_id:product_id' (hexadeciaml numbers) or
1015  'spice'
1016
1017 You can use the 'lsusb -t' command to list existing usb devices.
1018
1019 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1020 machines - use with special care.
1021
1022 The value 'spice' can be used to add a usb redirection devices for spice.
1023 EODESCR
1024     },
1025     usb3 => {
1026         optional => 1,
1027         type => 'boolean',
1028         description => "Specifies whether if given host option is a USB3 device or port.",
1029         default => 0,
1030     },
1031 };
1032
1033 my $usbdesc = {
1034     optional => 1,
1035     type => 'string', format => $usb_fmt,
1036     description => "Configure an USB device (n is 0 to 4).",
1037 };
1038 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1039
1040 my $serialdesc = {
1041         optional => 1,
1042         type => 'string',
1043         pattern => '(/dev/.+|socket)',
1044         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1045         verbose_description =>  <<EODESCR,
1046 Create a serial device inside the VM (n is 0 to 3), and pass through a
1047 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1048 host side (use 'qm terminal' to open a terminal connection).
1049
1050 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1051 use with special care.
1052
1053 CAUTION: Experimental! User reported problems with this option.
1054 EODESCR
1055 };
1056
1057 my $paralleldesc= {
1058         optional => 1,
1059         type => 'string',
1060         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1061         description =>  "Map host parallel devices (n is 0 to 2).",
1062         verbose_description =>  <<EODESCR,
1063 Map host parallel devices (n is 0 to 2).
1064
1065 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1066 machines - use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1073     $confdesc->{"parallel$i"} = $paralleldesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1077     $confdesc->{"serial$i"} = $serialdesc;
1078 }
1079
1080 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1081     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1082 }
1083
1084 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1085     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1086 }
1087
1088 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1089     $confdesc->{"usb$i"} = $usbdesc;
1090 }
1091
1092 my $boot_fmt = {
1093     legacy => {
1094         optional => 1,
1095         default_key => 1,
1096         type => 'string',
1097         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1098                      . " Deprecated, use 'order=' instead.",
1099         pattern => '[acdn]{1,4}',
1100         format_description => "[acdn]{1,4}",
1101
1102         # note: this is also the fallback if boot: is not given at all
1103         default => 'cdn',
1104     },
1105     order => {
1106         optional => 1,
1107         type => 'string',
1108         format => 'pve-qm-bootdev-list',
1109         format_description => "device[;device...]",
1110         description => <<EODESC,
1111 The guest will attempt to boot from devices in the order they appear here.
1112
1113 Disks, optical drives and passed-through storage USB devices will be directly
1114 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1115 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1116
1117 Note that only devices in this list will be marked as bootable and thus loaded
1118 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1119 (e.g. software-raid), you need to specify all of them here.
1120
1121 Overrides the deprecated 'legacy=[acdn]*' value when given.
1122 EODESC
1123     },
1124 };
1125 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1126
1127 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1128 sub verify_bootdev {
1129     my ($dev, $noerr) = @_;
1130
1131     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1132
1133     my $check = sub {
1134         my ($base) = @_;
1135         return 0 if $dev !~ m/^$base\d+$/;
1136         return 0 if !$confdesc->{$dev};
1137         return 1;
1138     };
1139
1140     return $dev if $check->("net");
1141     return $dev if $check->("usb");
1142     return $dev if $check->("hostpci");
1143
1144     return if $noerr;
1145     die "invalid boot device '$dev'\n";
1146 }
1147
1148 sub print_bootorder {
1149     my ($devs) = @_;
1150     return "" if !@$devs;
1151     my $data = { order => join(';', @$devs) };
1152     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1153 }
1154
1155 my $kvm_api_version = 0;
1156
1157 sub kvm_version {
1158     return $kvm_api_version if $kvm_api_version;
1159
1160     open my $fh, '<', '/dev/kvm' or return;
1161
1162     # 0xae00 => KVM_GET_API_VERSION
1163     $kvm_api_version = ioctl($fh, 0xae00, 0);
1164     close($fh);
1165
1166     return $kvm_api_version;
1167 }
1168
1169 my $kvm_user_version = {};
1170 my $kvm_mtime = {};
1171
1172 sub kvm_user_version {
1173     my ($binary) = @_;
1174
1175     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1176     my $st = stat($binary);
1177
1178     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180         $cachedmtime == $st->mtime;
1181
1182     $kvm_user_version->{$binary} = 'unknown';
1183     $kvm_mtime->{$binary} = $st->mtime;
1184
1185     my $code = sub {
1186         my $line = shift;
1187         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188             $kvm_user_version->{$binary} = $2;
1189         }
1190     };
1191
1192     eval { run_command([$binary, '--version'], outfunc => $code); };
1193     warn $@ if $@;
1194
1195     return $kvm_user_version->{$binary};
1196
1197 }
1198 my sub extract_version {
1199     my ($machine_type, $version) = @_;
1200     $version = kvm_user_version() if !defined($version);
1201     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1202 }
1203
1204 sub kernel_has_vhost_net {
1205     return -c '/dev/vhost-net';
1206 }
1207
1208 sub option_exists {
1209     my $key = shift;
1210     return defined($confdesc->{$key});
1211 }
1212
1213 my $cdrom_path;
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241      if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1286             if !$vtype;
1287         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1288         verify_media_type($opt, $vtype, $drive->{media});
1289         $drive->{file} = $volid;
1290     }
1291
1292     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1293 }
1294
1295 sub parse_hotplug_features {
1296     my ($data) = @_;
1297
1298     my $res = {};
1299
1300     return $res if $data eq '0';
1301
1302     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1303
1304     foreach my $feature (PVE::Tools::split_list($data)) {
1305         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306             $res->{$1} = 1;
1307         } else {
1308             die "invalid hotplug feature '$feature'\n";
1309         }
1310     }
1311     return $res;
1312 }
1313
1314 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1315 sub pve_verify_hotplug_features {
1316     my ($value, $noerr) = @_;
1317
1318     return $value if parse_hotplug_features($value);
1319
1320     return if $noerr;
1321
1322     die "unable to parse hotplug option\n";
1323 }
1324
1325 sub scsi_inquiry {
1326     my($fh, $noerr) = @_;
1327
1328     my $SG_IO = 0x2285;
1329     my $SG_GET_VERSION_NUM = 0x2282;
1330
1331     my $versionbuf = "\x00" x 8;
1332     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333     if (!$ret) {
1334         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335         return;
1336     }
1337     my $version = unpack("I", $versionbuf);
1338     if ($version < 30000) {
1339         die "scsi generic interface too old\n"  if !$noerr;
1340         return;
1341     }
1342
1343     my $buf = "\x00" x 36;
1344     my $sensebuf = "\x00" x 8;
1345     my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347     # see /usr/include/scsi/sg.h
1348     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351                       length($sensebuf), 0, length($buf), $buf,
1352                       $cmd, $sensebuf, 6000);
1353
1354     $ret = ioctl($fh, $SG_IO, $packet);
1355     if (!$ret) {
1356         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357         return;
1358     }
1359
1360     my @res = unpack($sg_io_hdr_t, $packet);
1361     if ($res[17] || $res[18]) {
1362         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363         return;
1364     }
1365
1366     my $res = {};
1367     (my $byte0, my $byte1, $res->{vendor},
1368      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370     $res->{removable} = $byte1 & 128 ? 1 : 0;
1371     $res->{type} = $byte0 & 31;
1372
1373     return $res;
1374 }
1375
1376 sub path_is_scsi {
1377     my ($path) = @_;
1378
1379     my $fh = IO::File->new("+<$path") || return;
1380     my $res = scsi_inquiry($fh, 1);
1381     close($fh);
1382
1383     return $res;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf, $arch) = @_;
1388
1389     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus;
1393     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1394         $usbbus = 'ehci';
1395     } else {
1396         $usbbus = 'uhci';
1397     }
1398
1399     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400 }
1401
1402 sub print_keyboarddevice_full {
1403     my ($conf, $arch, $machine) = @_;
1404
1405     return if $arch ne 'aarch64';
1406
1407     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408 }
1409
1410 my sub get_drive_id {
1411     my ($drive) = @_;
1412     return "$drive->{interface}$drive->{index}";
1413 }
1414
1415 sub print_drivedevice_full {
1416     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1417
1418     my $device = '';
1419     my $maxdev = 0;
1420
1421     my $drive_id = get_drive_id($drive);
1422     if ($drive->{interface} eq 'virtio') {
1423         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1424         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1425         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1426     } elsif ($drive->{interface} eq 'scsi') {
1427
1428         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1429         my $unit = $drive->{index} % $maxdev;
1430         my $devicetype = 'hd';
1431         my $path = '';
1432         if (drive_is_cdrom($drive)) {
1433             $devicetype = 'cd';
1434         } else {
1435             if ($drive->{file} =~ m|^/|) {
1436                 $path = $drive->{file};
1437                 if (my $info = path_is_scsi($path)) {
1438                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1439                         $devicetype = 'block';
1440                     } elsif ($info->{type} == 1) { # tape
1441                         $devicetype = 'generic';
1442                     }
1443                 }
1444             } else {
1445                  $path = PVE::Storage::path($storecfg, $drive->{file});
1446             }
1447
1448             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1449             my $version = extract_version($machine_type, kvm_user_version());
1450             if ($path =~ m/^iscsi\:\/\// &&
1451                !min_version($version, 4, 1)) {
1452                 $devicetype = 'generic';
1453             }
1454         }
1455
1456         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1458         } else {
1459             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1460                 .",lun=$drive->{index}";
1461         }
1462         $device .= ",drive=drive-$drive_id,id=$drive_id";
1463
1464         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1465             $device .= ",rotation_rate=1";
1466         }
1467         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1468
1469     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1470         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1471         my $controller = int($drive->{index} / $maxdev);
1472         my $unit = $drive->{index} % $maxdev;
1473         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1474
1475         $device = "ide-$devicetype";
1476         if ($drive->{interface} eq 'ide') {
1477             $device .= ",bus=ide.$controller,unit=$unit";
1478         } else {
1479             $device .= ",bus=ahci$controller.$unit";
1480         }
1481         $device .= ",drive=drive-$drive_id,id=$drive_id";
1482
1483         if ($devicetype eq 'hd') {
1484             if (my $model = $drive->{model}) {
1485                 $model = URI::Escape::uri_unescape($model);
1486                 $device .= ",model=$model";
1487             }
1488             if ($drive->{ssd}) {
1489                 $device .= ",rotation_rate=1";
1490             }
1491         }
1492         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1493     } elsif ($drive->{interface} eq 'usb') {
1494         die "implement me";
1495         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1496     } else {
1497         die "unsupported interface type";
1498     }
1499
1500     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1501
1502     if (my $serial = $drive->{serial}) {
1503         $serial = URI::Escape::uri_unescape($serial);
1504         $device .= ",serial=$serial";
1505     }
1506
1507
1508     return $device;
1509 }
1510
1511 sub get_initiator_name {
1512     my $initiator;
1513
1514     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1515     while (defined(my $line = <$fh>)) {
1516         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1517         $initiator = $1;
1518         last;
1519     }
1520     $fh->close();
1521
1522     return $initiator;
1523 }
1524
1525 sub print_drive_commandline_full {
1526     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1527
1528     my $path;
1529     my $volid = $drive->{file};
1530     my $format = $drive->{format};
1531     my $drive_id = get_drive_id($drive);
1532
1533     if (drive_is_cdrom($drive)) {
1534         $path = get_iso_path($storecfg, $vmid, $volid);
1535         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1536     } else {
1537         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1538         if ($storeid) {
1539             $path = PVE::Storage::path($storecfg, $volid);
1540             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1541             $format //= qemu_img_format($scfg, $volname);
1542         } else {
1543             $path = $volid;
1544             $format //= "raw";
1545         }
1546    }
1547
1548    my $is_rbd = $path =~ m/^rbd:/;
1549
1550     my $opts = '';
1551     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1552     foreach my $o (@qemu_drive_options) {
1553         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1554     }
1555
1556     # snapshot only accepts on|off
1557     if (defined($drive->{snapshot})) {
1558         my $v = $drive->{snapshot} ? 'on' : 'off';
1559         $opts .= ",snapshot=$v";
1560     }
1561
1562     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1563         my ($dir, $qmpname) = @$type;
1564         if (my $v = $drive->{"mbps$dir"}) {
1565             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1566         }
1567         if (my $v = $drive->{"mbps${dir}_max"}) {
1568             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1569         }
1570         if (my $v = $drive->{"bps${dir}_max_length"}) {
1571             $opts .= ",throttling.bps$qmpname-max-length=$v";
1572         }
1573         if (my $v = $drive->{"iops${dir}"}) {
1574             $opts .= ",throttling.iops$qmpname=$v";
1575         }
1576         if (my $v = $drive->{"iops${dir}_max"}) {
1577             $opts .= ",throttling.iops$qmpname-max=$v";
1578         }
1579         if (my $v = $drive->{"iops${dir}_max_length"}) {
1580             $opts .= ",throttling.iops$qmpname-max-length=$v";
1581         }
1582     }
1583
1584     if ($pbs_name) {
1585         $format = "rbd" if $is_rbd;
1586         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1587             if !$format;
1588         $opts .= ",format=alloc-track,file.driver=$format";
1589     } elsif ($format) {
1590         $opts .= ",format=$format";
1591     }
1592
1593     my $cache_direct = 0;
1594
1595     if (my $cache = $drive->{cache}) {
1596         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1597     } elsif (!drive_is_cdrom($drive)) {
1598         $opts .= ",cache=none";
1599         $cache_direct = 1;
1600     }
1601
1602     if (!$drive->{aio}) {
1603         if ($io_uring) {
1604             # io_uring supports all cache modes
1605             $opts .= ",aio=io_uring";
1606         } else {
1607             # aio native works only with O_DIRECT
1608             if($cache_direct) {
1609                 $opts .= ",aio=native";
1610             } else {
1611                 $opts .= ",aio=threads";
1612             }
1613         }
1614     }
1615
1616     if (!drive_is_cdrom($drive)) {
1617         my $detectzeroes;
1618         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1619             $detectzeroes = 'off';
1620         } elsif ($drive->{discard}) {
1621             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1622         } else {
1623             # This used to be our default with discard not being specified:
1624             $detectzeroes = 'on';
1625         }
1626
1627         # note: 'detect-zeroes' works per blockdev and we want it to persist
1628         # after the alloc-track is removed, so put it on 'file' directly
1629         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1630         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1631     }
1632
1633     if ($pbs_name) {
1634         $opts .= ",backing=$pbs_name";
1635         $opts .= ",auto-remove=on";
1636     }
1637
1638     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1639     my $file_param = "file";
1640     if ($pbs_name) {
1641         # non-rbd drivers require the underlying file to be a seperate block
1642         # node, so add a second .file indirection
1643         $file_param .= ".file" if !$is_rbd;
1644         $file_param .= ".filename";
1645     }
1646     my $pathinfo = $path ? "$file_param=$path," : '';
1647
1648     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1649 }
1650
1651 sub print_pbs_blockdev {
1652     my ($pbs_conf, $pbs_name) = @_;
1653     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1654     $blockdev .= ",repository=$pbs_conf->{repository}";
1655     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1656     $blockdev .= ",archive=$pbs_conf->{archive}";
1657     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1658     return $blockdev;
1659 }
1660
1661 sub print_netdevice_full {
1662     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1663
1664     my $device = $net->{model};
1665     if ($net->{model} eq 'virtio') {
1666          $device = 'virtio-net-pci';
1667      };
1668
1669     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1670     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1671     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1672         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1673         # and out of each queue plus one config interrupt and control vector queue
1674         my $vectors = $net->{queues} * 2 + 2;
1675         $tmpstr .= ",vectors=$vectors,mq=on";
1676     }
1677     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1678
1679     if (my $mtu = $net->{mtu}) {
1680         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1681             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1682             if ($mtu == 1) {
1683                  $mtu = $bridge_mtu;
1684             } elsif ($mtu < 576) {
1685                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1686             } elsif ($mtu > $bridge_mtu) {
1687                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1688             }
1689             $tmpstr .= ",host_mtu=$mtu";
1690         } else {
1691             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1692         }
1693     }
1694
1695     if ($use_old_bios_files) {
1696         my $romfile;
1697         if ($device eq 'virtio-net-pci') {
1698             $romfile = 'pxe-virtio.rom';
1699         } elsif ($device eq 'e1000') {
1700             $romfile = 'pxe-e1000.rom';
1701         } elsif ($device eq 'ne2k') {
1702             $romfile = 'pxe-ne2k_pci.rom';
1703         } elsif ($device eq 'pcnet') {
1704             $romfile = 'pxe-pcnet.rom';
1705         } elsif ($device eq 'rtl8139') {
1706             $romfile = 'pxe-rtl8139.rom';
1707         }
1708         $tmpstr .= ",romfile=$romfile" if $romfile;
1709     }
1710
1711     return $tmpstr;
1712 }
1713
1714 sub print_netdev_full {
1715     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1716
1717     my $i = '';
1718     if ($netid =~ m/^net(\d+)$/) {
1719         $i = int($1);
1720     }
1721
1722     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1723
1724     my $ifname = "tap${vmid}i$i";
1725
1726     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1727     die "interface name '$ifname' is too long (max 15 character)\n"
1728         if length($ifname) >= 16;
1729
1730     my $vhostparam = '';
1731     if (is_native($arch)) {
1732         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1733     }
1734
1735     my $vmname = $conf->{name} || "vm$vmid";
1736
1737     my $netdev = "";
1738     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1739
1740     if ($net->{bridge}) {
1741         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1742             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1743     } else {
1744         $netdev = "type=user,id=$netid,hostname=$vmname";
1745     }
1746
1747     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1748
1749     return $netdev;
1750 }
1751
1752 my $vga_map = {
1753     'cirrus' => 'cirrus-vga',
1754     'std' => 'VGA',
1755     'vmware' => 'vmware-svga',
1756     'virtio' => 'virtio-vga',
1757 };
1758
1759 sub print_vga_device {
1760     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1761
1762     my $type = $vga_map->{$vga->{type}};
1763     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1764         $type = 'virtio-gpu';
1765     }
1766     my $vgamem_mb = $vga->{memory};
1767
1768     my $max_outputs = '';
1769     if ($qxlnum) {
1770         $type = $id ? 'qxl' : 'qxl-vga';
1771
1772         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1773             # set max outputs so linux can have up to 4 qxl displays with one device
1774             if (min_version($machine_version, 4, 1)) {
1775                 $max_outputs = ",max_outputs=4";
1776             }
1777         }
1778     }
1779
1780     die "no devicetype for $vga->{type}\n" if !$type;
1781
1782     my $memory = "";
1783     if ($vgamem_mb) {
1784         if ($vga->{type} eq 'virtio') {
1785             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1786             $memory = ",max_hostmem=$bytes";
1787         } elsif ($qxlnum) {
1788             # from https://www.spice-space.org/multiple-monitors.html
1789             $memory = ",vgamem_mb=$vga->{memory}";
1790             my $ram = $vgamem_mb * 4;
1791             my $vram = $vgamem_mb * 2;
1792             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1793         } else {
1794             $memory = ",vgamem_mb=$vga->{memory}";
1795         }
1796     } elsif ($qxlnum && $id) {
1797         $memory = ",ram_size=67108864,vram_size=33554432";
1798     }
1799
1800     my $edidoff = "";
1801     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1802         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1803     }
1804
1805     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1806     my $vgaid = "vga" . ($id // '');
1807     my $pciaddr;
1808
1809     if ($q35 && $vgaid eq 'vga') {
1810         # the first display uses pcie.0 bus on q35 machines
1811         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1812     } else {
1813         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1814     }
1815
1816     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1817 }
1818
1819 sub parse_number_sets {
1820     my ($set) = @_;
1821     my $res = [];
1822     foreach my $part (split(/;/, $set)) {
1823         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1824             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1825             push @$res, [ $1, $2 ];
1826         } else {
1827             die "invalid range: $part\n";
1828         }
1829     }
1830     return $res;
1831 }
1832
1833 sub parse_numa {
1834     my ($data) = @_;
1835
1836     my $res = parse_property_string($numa_fmt, $data);
1837     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1838     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1839     return $res;
1840 }
1841
1842 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1843 sub parse_net {
1844     my ($data) = @_;
1845
1846     my $res = eval { parse_property_string($net_fmt, $data) };
1847     if ($@) {
1848         warn $@;
1849         return;
1850     }
1851     if (!defined($res->{macaddr})) {
1852         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1853         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1854     }
1855     return $res;
1856 }
1857
1858 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1859 sub parse_ipconfig {
1860     my ($data) = @_;
1861
1862     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1863     if ($@) {
1864         warn $@;
1865         return;
1866     }
1867
1868     if ($res->{gw} && !$res->{ip}) {
1869         warn 'gateway specified without specifying an IP address';
1870         return;
1871     }
1872     if ($res->{gw6} && !$res->{ip6}) {
1873         warn 'IPv6 gateway specified without specifying an IPv6 address';
1874         return;
1875     }
1876     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1877         warn 'gateway specified together with DHCP';
1878         return;
1879     }
1880     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1881         # gw6 + auto/dhcp
1882         warn "IPv6 gateway specified together with $res->{ip6} address";
1883         return;
1884     }
1885
1886     if (!$res->{ip} && !$res->{ip6}) {
1887         return { ip => 'dhcp', ip6 => 'dhcp' };
1888     }
1889
1890     return $res;
1891 }
1892
1893 sub print_net {
1894     my $net = shift;
1895
1896     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1897 }
1898
1899 sub add_random_macs {
1900     my ($settings) = @_;
1901
1902     foreach my $opt (keys %$settings) {
1903         next if $opt !~ m/^net(\d+)$/;
1904         my $net = parse_net($settings->{$opt});
1905         next if !$net;
1906         $settings->{$opt} = print_net($net);
1907     }
1908 }
1909
1910 sub vm_is_volid_owner {
1911     my ($storecfg, $vmid, $volid) = @_;
1912
1913     if ($volid !~  m|^/|) {
1914         my ($path, $owner);
1915         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1916         if ($owner && ($owner == $vmid)) {
1917             return 1;
1918         }
1919     }
1920
1921     return;
1922 }
1923
1924 sub vmconfig_register_unused_drive {
1925     my ($storecfg, $vmid, $conf, $drive) = @_;
1926
1927     if (drive_is_cloudinit($drive)) {
1928         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1929         warn $@ if $@;
1930     } elsif (!drive_is_cdrom($drive)) {
1931         my $volid = $drive->{file};
1932         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1933             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1934         }
1935     }
1936 }
1937
1938 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1939 my $smbios1_fmt = {
1940     uuid => {
1941         type => 'string',
1942         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1943         format_description => 'UUID',
1944         description => "Set SMBIOS1 UUID.",
1945         optional => 1,
1946     },
1947     version => {
1948         type => 'string',
1949         pattern => '[A-Za-z0-9+\/]+={0,2}',
1950         format_description => 'Base64 encoded string',
1951         description => "Set SMBIOS1 version.",
1952         optional => 1,
1953     },
1954     serial => {
1955         type => 'string',
1956         pattern => '[A-Za-z0-9+\/]+={0,2}',
1957         format_description => 'Base64 encoded string',
1958         description => "Set SMBIOS1 serial number.",
1959         optional => 1,
1960     },
1961     manufacturer => {
1962         type => 'string',
1963         pattern => '[A-Za-z0-9+\/]+={0,2}',
1964         format_description => 'Base64 encoded string',
1965         description => "Set SMBIOS1 manufacturer.",
1966         optional => 1,
1967     },
1968     product => {
1969         type => 'string',
1970         pattern => '[A-Za-z0-9+\/]+={0,2}',
1971         format_description => 'Base64 encoded string',
1972         description => "Set SMBIOS1 product ID.",
1973         optional => 1,
1974     },
1975     sku => {
1976         type => 'string',
1977         pattern => '[A-Za-z0-9+\/]+={0,2}',
1978         format_description => 'Base64 encoded string',
1979         description => "Set SMBIOS1 SKU string.",
1980         optional => 1,
1981     },
1982     family => {
1983         type => 'string',
1984         pattern => '[A-Za-z0-9+\/]+={0,2}',
1985         format_description => 'Base64 encoded string',
1986         description => "Set SMBIOS1 family string.",
1987         optional => 1,
1988     },
1989     base64 => {
1990         type => 'boolean',
1991         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1992         optional => 1,
1993     },
1994 };
1995
1996 sub parse_smbios1 {
1997     my ($data) = @_;
1998
1999     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2000     warn $@ if $@;
2001     return $res;
2002 }
2003
2004 sub print_smbios1 {
2005     my ($smbios1) = @_;
2006     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2010
2011 sub parse_watchdog {
2012     my ($value) = @_;
2013
2014     return if !$value;
2015
2016     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2017     warn $@ if $@;
2018     return $res;
2019 }
2020
2021 sub parse_guest_agent {
2022     my ($conf) = @_;
2023
2024     return {} if !defined($conf->{agent});
2025
2026     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2027     warn $@ if $@;
2028
2029     # if the agent is disabled ignore the other potentially set properties
2030     return {} if !$res->{enabled};
2031     return $res;
2032 }
2033
2034 sub get_qga_key {
2035     my ($conf, $key) = @_;
2036     return undef if !defined($conf->{agent});
2037
2038     my $agent = parse_guest_agent($conf);
2039     return $agent->{$key};
2040 }
2041
2042 sub parse_vga {
2043     my ($value) = @_;
2044
2045     return {} if !$value;
2046     my $res = eval { parse_property_string($vga_fmt, $value) };
2047     warn $@ if $@;
2048     return $res;
2049 }
2050
2051 sub parse_rng {
2052     my ($value) = @_;
2053
2054     return if !$value;
2055
2056     my $res = eval { parse_property_string($rng_fmt, $value) };
2057     warn $@ if $@;
2058     return $res;
2059 }
2060
2061 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2062 sub verify_usb_device {
2063     my ($value, $noerr) = @_;
2064
2065     return $value if parse_usb_device($value);
2066
2067     return if $noerr;
2068
2069     die "unable to parse usb device\n";
2070 }
2071
2072 # add JSON properties for create and set function
2073 sub json_config_properties {
2074     my $prop = shift;
2075
2076     foreach my $opt (keys %$confdesc) {
2077         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2078             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2079         $prop->{$opt} = $confdesc->{$opt};
2080     }
2081
2082     return $prop;
2083 }
2084
2085 # return copy of $confdesc_cloudinit to generate documentation
2086 sub cloudinit_config_properties {
2087
2088     return dclone($confdesc_cloudinit);
2089 }
2090
2091 sub check_type {
2092     my ($key, $value) = @_;
2093
2094     die "unknown setting '$key'\n" if !$confdesc->{$key};
2095
2096     my $type = $confdesc->{$key}->{type};
2097
2098     if (!defined($value)) {
2099         die "got undefined value\n";
2100     }
2101
2102     if ($value =~ m/[\n\r]/) {
2103         die "property contains a line feed\n";
2104     }
2105
2106     if ($type eq 'boolean') {
2107         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2108         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2109         die "type check ('boolean') failed - got '$value'\n";
2110     } elsif ($type eq 'integer') {
2111         return int($1) if $value =~ m/^(\d+)$/;
2112         die "type check ('integer') failed - got '$value'\n";
2113     } elsif ($type eq 'number') {
2114         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2115         die "type check ('number') failed - got '$value'\n";
2116     } elsif ($type eq 'string') {
2117         if (my $fmt = $confdesc->{$key}->{format}) {
2118             PVE::JSONSchema::check_format($fmt, $value);
2119             return $value;
2120         }
2121         $value =~ s/^\"(.*)\"$/$1/;
2122         return $value;
2123     } else {
2124         die "internal error"
2125     }
2126 }
2127
2128 sub destroy_vm {
2129     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2130
2131     my $conf = PVE::QemuConfig->load_config($vmid);
2132
2133     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2134
2135     if ($conf->{template}) {
2136         # check if any base image is still used by a linked clone
2137         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2138                 my ($ds, $drive) = @_;
2139                 return if drive_is_cdrom($drive);
2140
2141                 my $volid = $drive->{file};
2142                 return if !$volid || $volid =~ m|^/|;
2143
2144                 die "base volume '$volid' is still in use by linked cloned\n"
2145                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2146
2147         });
2148     }
2149
2150     my $remove_owned_drive = sub {
2151         my ($ds, $drive) = @_;
2152         return if drive_is_cdrom($drive, 1);
2153
2154         my $volid = $drive->{file};
2155         return if !$volid || $volid =~ m|^/|;
2156
2157         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2158         return if !$path || !$owner || ($owner != $vmid);
2159
2160         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2161         warn "Could not remove disk '$volid', check manually: $@" if $@;
2162     };
2163
2164     # only remove disks owned by this VM (referenced in the config)
2165     my $include_opts = {
2166         include_unused => 1,
2167         extra_keys => ['vmstate'],
2168     };
2169     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2170
2171     for my $snap (values %{$conf->{snapshots}}) {
2172         next if !defined($snap->{vmstate});
2173         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2174         next if !defined($drive);
2175         $remove_owned_drive->('vmstate', $drive);
2176     }
2177
2178     if ($purge_unreferenced) { # also remove unreferenced disk
2179         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2180         PVE::Storage::foreach_volid($vmdisks, sub {
2181             my ($volid, $sid, $volname, $d) = @_;
2182             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2183             warn $@ if $@;
2184         });
2185     }
2186
2187     if (defined $replacement_conf) {
2188         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2189     } else {
2190         PVE::QemuConfig->destroy_config($vmid);
2191     }
2192 }
2193
2194 sub parse_vm_config {
2195     my ($filename, $raw) = @_;
2196
2197     return if !defined($raw);
2198
2199     my $res = {
2200         digest => Digest::SHA::sha1_hex($raw),
2201         snapshots => {},
2202         pending => {},
2203     };
2204
2205     $filename =~ m|/qemu-server/(\d+)\.conf$|
2206         || die "got strange filename '$filename'";
2207
2208     my $vmid = $1;
2209
2210     my $conf = $res;
2211     my $descr;
2212     my $section = '';
2213
2214     my @lines = split(/\n/, $raw);
2215     foreach my $line (@lines) {
2216         next if $line =~ m/^\s*$/;
2217
2218         if ($line =~ m/^\[PENDING\]\s*$/i) {
2219             $section = 'pending';
2220             if (defined($descr)) {
2221                 $descr =~ s/\s+$//;
2222                 $conf->{description} = $descr;
2223             }
2224             $descr = undef;
2225             $conf = $res->{$section} = {};
2226             next;
2227
2228         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2229             $section = $1;
2230             if (defined($descr)) {
2231                 $descr =~ s/\s+$//;
2232                 $conf->{description} = $descr;
2233             }
2234             $descr = undef;
2235             $conf = $res->{snapshots}->{$section} = {};
2236             next;
2237         }
2238
2239         if ($line =~ m/^\#(.*)\s*$/) {
2240             $descr = '' if !defined($descr);
2241             $descr .= PVE::Tools::decode_text($1) . "\n";
2242             next;
2243         }
2244
2245         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2246             $descr = '' if !defined($descr);
2247             $descr .= PVE::Tools::decode_text($2);
2248         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2249             $conf->{snapstate} = $1;
2250         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2251             my $key = $1;
2252             my $value = $2;
2253             $conf->{$key} = $value;
2254         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2255             my $value = $1;
2256             if ($section eq 'pending') {
2257                 $conf->{delete} = $value; # we parse this later
2258             } else {
2259                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2260             }
2261         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2262             my $key = $1;
2263             my $value = $2;
2264             eval { $value = check_type($key, $value); };
2265             if ($@) {
2266                 warn "vm $vmid - unable to parse value of '$key' - $@";
2267             } else {
2268                 $key = 'ide2' if $key eq 'cdrom';
2269                 my $fmt = $confdesc->{$key}->{format};
2270                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2271                     my $v = parse_drive($key, $value);
2272                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2273                         $v->{file} = $volid;
2274                         $value = print_drive($v);
2275                     } else {
2276                         warn "vm $vmid - unable to parse value of '$key'\n";
2277                         next;
2278                     }
2279                 }
2280
2281                 $conf->{$key} = $value;
2282             }
2283         } else {
2284             warn "vm $vmid - unable to parse config: $line\n";
2285         }
2286     }
2287
2288     if (defined($descr)) {
2289         $descr =~ s/\s+$//;
2290         $conf->{description} = $descr;
2291     }
2292     delete $res->{snapstate}; # just to be sure
2293
2294     return $res;
2295 }
2296
2297 sub write_vm_config {
2298     my ($filename, $conf) = @_;
2299
2300     delete $conf->{snapstate}; # just to be sure
2301
2302     if ($conf->{cdrom}) {
2303         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2304         $conf->{ide2} = $conf->{cdrom};
2305         delete $conf->{cdrom};
2306     }
2307
2308     # we do not use 'smp' any longer
2309     if ($conf->{sockets}) {
2310         delete $conf->{smp};
2311     } elsif ($conf->{smp}) {
2312         $conf->{sockets} = $conf->{smp};
2313         delete $conf->{cores};
2314         delete $conf->{smp};
2315     }
2316
2317     my $used_volids = {};
2318
2319     my $cleanup_config = sub {
2320         my ($cref, $pending, $snapname) = @_;
2321
2322         foreach my $key (keys %$cref) {
2323             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2324                 $key eq 'snapstate' || $key eq 'pending';
2325             my $value = $cref->{$key};
2326             if ($key eq 'delete') {
2327                 die "propertry 'delete' is only allowed in [PENDING]\n"
2328                     if !$pending;
2329                 # fixme: check syntax?
2330                 next;
2331             }
2332             eval { $value = check_type($key, $value); };
2333             die "unable to parse value of '$key' - $@" if $@;
2334
2335             $cref->{$key} = $value;
2336
2337             if (!$snapname && is_valid_drivename($key)) {
2338                 my $drive = parse_drive($key, $value);
2339                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2340             }
2341         }
2342     };
2343
2344     &$cleanup_config($conf);
2345
2346     &$cleanup_config($conf->{pending}, 1);
2347
2348     foreach my $snapname (keys %{$conf->{snapshots}}) {
2349         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2350         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2351     }
2352
2353     # remove 'unusedX' settings if we re-add a volume
2354     foreach my $key (keys %$conf) {
2355         my $value = $conf->{$key};
2356         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2357             delete $conf->{$key};
2358         }
2359     }
2360
2361     my $generate_raw_config = sub {
2362         my ($conf, $pending) = @_;
2363
2364         my $raw = '';
2365
2366         # add description as comment to top of file
2367         if (defined(my $descr = $conf->{description})) {
2368             if ($descr) {
2369                 foreach my $cl (split(/\n/, $descr)) {
2370                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2371                 }
2372             } else {
2373                 $raw .= "#\n" if $pending;
2374             }
2375         }
2376
2377         foreach my $key (sort keys %$conf) {
2378             next if $key =~ /^(digest|description|pending|snapshots)$/;
2379             $raw .= "$key: $conf->{$key}\n";
2380         }
2381         return $raw;
2382     };
2383
2384     my $raw = &$generate_raw_config($conf);
2385
2386     if (scalar(keys %{$conf->{pending}})){
2387         $raw .= "\n[PENDING]\n";
2388         $raw .= &$generate_raw_config($conf->{pending}, 1);
2389     }
2390
2391     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2392         $raw .= "\n[$snapname]\n";
2393         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2394     }
2395
2396     return $raw;
2397 }
2398
2399 sub load_defaults {
2400
2401     my $res = {};
2402
2403     # we use static defaults from our JSON schema configuration
2404     foreach my $key (keys %$confdesc) {
2405         if (defined(my $default = $confdesc->{$key}->{default})) {
2406             $res->{$key} = $default;
2407         }
2408     }
2409
2410     return $res;
2411 }
2412
2413 sub config_list {
2414     my $vmlist = PVE::Cluster::get_vmlist();
2415     my $res = {};
2416     return $res if !$vmlist || !$vmlist->{ids};
2417     my $ids = $vmlist->{ids};
2418     my $nodename = nodename();
2419
2420     foreach my $vmid (keys %$ids) {
2421         my $d = $ids->{$vmid};
2422         next if !$d->{node} || $d->{node} ne $nodename;
2423         next if !$d->{type} || $d->{type} ne 'qemu';
2424         $res->{$vmid}->{exists} = 1;
2425     }
2426     return $res;
2427 }
2428
2429 # test if VM uses local resources (to prevent migration)
2430 sub check_local_resources {
2431     my ($conf, $noerr) = @_;
2432
2433     my @loc_res = ();
2434
2435     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2436     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2437
2438     push @loc_res, "ivshmem" if $conf->{ivshmem};
2439
2440     foreach my $k (keys %$conf) {
2441         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2442         # sockets are safe: they will recreated be on the target side post-migrate
2443         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2444         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2445     }
2446
2447     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2448
2449     return \@loc_res;
2450 }
2451
2452 # check if used storages are available on all nodes (use by migrate)
2453 sub check_storage_availability {
2454     my ($storecfg, $conf, $node) = @_;
2455
2456     PVE::QemuConfig->foreach_volume($conf, sub {
2457         my ($ds, $drive) = @_;
2458
2459         my $volid = $drive->{file};
2460         return if !$volid;
2461
2462         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2463         return if !$sid;
2464
2465         # check if storage is available on both nodes
2466         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2467         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2468
2469         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2470
2471         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2472             if !$scfg->{content}->{$vtype};
2473    });
2474 }
2475
2476 # list nodes where all VM images are available (used by has_feature API)
2477 sub shared_nodes {
2478     my ($conf, $storecfg) = @_;
2479
2480     my $nodelist = PVE::Cluster::get_nodelist();
2481     my $nodehash = { map { $_ => 1 } @$nodelist };
2482     my $nodename = nodename();
2483
2484     PVE::QemuConfig->foreach_volume($conf, sub {
2485         my ($ds, $drive) = @_;
2486
2487         my $volid = $drive->{file};
2488         return if !$volid;
2489
2490         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2491         if ($storeid) {
2492             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2493             if ($scfg->{disable}) {
2494                 $nodehash = {};
2495             } elsif (my $avail = $scfg->{nodes}) {
2496                 foreach my $node (keys %$nodehash) {
2497                     delete $nodehash->{$node} if !$avail->{$node};
2498                 }
2499             } elsif (!$scfg->{shared}) {
2500                 foreach my $node (keys %$nodehash) {
2501                     delete $nodehash->{$node} if $node ne $nodename
2502                 }
2503             }
2504         }
2505     });
2506
2507     return $nodehash
2508 }
2509
2510 sub check_local_storage_availability {
2511     my ($conf, $storecfg) = @_;
2512
2513     my $nodelist = PVE::Cluster::get_nodelist();
2514     my $nodehash = { map { $_ => {} } @$nodelist };
2515
2516     PVE::QemuConfig->foreach_volume($conf, sub {
2517         my ($ds, $drive) = @_;
2518
2519         my $volid = $drive->{file};
2520         return if !$volid;
2521
2522         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2523         if ($storeid) {
2524             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2525
2526             if ($scfg->{disable}) {
2527                 foreach my $node (keys %$nodehash) {
2528                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2529                 }
2530             } elsif (my $avail = $scfg->{nodes}) {
2531                 foreach my $node (keys %$nodehash) {
2532                     if (!$avail->{$node}) {
2533                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2534                     }
2535                 }
2536             }
2537         }
2538     });
2539
2540     foreach my $node (values %$nodehash) {
2541         if (my $unavail = $node->{unavailable_storages}) {
2542             $node->{unavailable_storages} = [ sort keys %$unavail ];
2543         }
2544     }
2545
2546     return $nodehash
2547 }
2548
2549 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2550 sub check_running {
2551     my ($vmid, $nocheck, $node) = @_;
2552
2553     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2554     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2555 }
2556
2557 sub vzlist {
2558
2559     my $vzlist = config_list();
2560
2561     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2562
2563     while (defined(my $de = $fd->read)) {
2564         next if $de !~ m/^(\d+)\.pid$/;
2565         my $vmid = $1;
2566         next if !defined($vzlist->{$vmid});
2567         if (my $pid = check_running($vmid)) {
2568             $vzlist->{$vmid}->{pid} = $pid;
2569         }
2570     }
2571
2572     return $vzlist;
2573 }
2574
2575 our $vmstatus_return_properties = {
2576     vmid => get_standard_option('pve-vmid'),
2577     status => {
2578         description => "Qemu process status.",
2579         type => 'string',
2580         enum => ['stopped', 'running'],
2581     },
2582     maxmem => {
2583         description => "Maximum memory in bytes.",
2584         type => 'integer',
2585         optional => 1,
2586         renderer => 'bytes',
2587     },
2588     maxdisk => {
2589         description => "Root disk size in bytes.",
2590         type => 'integer',
2591         optional => 1,
2592         renderer => 'bytes',
2593     },
2594     name => {
2595         description => "VM name.",
2596         type => 'string',
2597         optional => 1,
2598     },
2599     qmpstatus => {
2600         description => "Qemu QMP agent status.",
2601         type => 'string',
2602         optional => 1,
2603     },
2604     pid => {
2605         description => "PID of running qemu process.",
2606         type => 'integer',
2607         optional => 1,
2608     },
2609     uptime => {
2610         description => "Uptime.",
2611         type => 'integer',
2612         optional => 1,
2613         renderer => 'duration',
2614     },
2615     cpus => {
2616         description => "Maximum usable CPUs.",
2617         type => 'number',
2618         optional => 1,
2619     },
2620     lock => {
2621         description => "The current config lock, if any.",
2622         type => 'string',
2623         optional => 1,
2624     },
2625     tags => {
2626         description  => "The current configured tags, if any",
2627         type => 'string',
2628         optional => 1,
2629     },
2630     'running-machine' => {
2631         description => "The currently running machine type (if running).",
2632         type => 'string',
2633         optional => 1,
2634     },
2635     'running-qemu' => {
2636         description => "The currently running QEMU version (if running).",
2637         type => 'string',
2638         optional => 1,
2639     },
2640 };
2641
2642 my $last_proc_pid_stat;
2643
2644 # get VM status information
2645 # This must be fast and should not block ($full == false)
2646 # We only query KVM using QMP if $full == true (this can be slow)
2647 sub vmstatus {
2648     my ($opt_vmid, $full) = @_;
2649
2650     my $res = {};
2651
2652     my $storecfg = PVE::Storage::config();
2653
2654     my $list = vzlist();
2655     my $defaults = load_defaults();
2656
2657     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2658
2659     my $cpucount = $cpuinfo->{cpus} || 1;
2660
2661     foreach my $vmid (keys %$list) {
2662         next if $opt_vmid && ($vmid ne $opt_vmid);
2663
2664         my $conf = PVE::QemuConfig->load_config($vmid);
2665
2666         my $d = { vmid => int($vmid) };
2667         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2668
2669         # fixme: better status?
2670         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2671
2672         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2673         if (defined($size)) {
2674             $d->{disk} = 0; # no info available
2675             $d->{maxdisk} = $size;
2676         } else {
2677             $d->{disk} = 0;
2678             $d->{maxdisk} = 0;
2679         }
2680
2681         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2682             * ($conf->{cores} || $defaults->{cores});
2683         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2684         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2685
2686         $d->{name} = $conf->{name} || "VM $vmid";
2687         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2688             : $defaults->{memory}*(1024*1024);
2689
2690         if ($conf->{balloon}) {
2691             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2692             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2693                 : $defaults->{shares};
2694         }
2695
2696         $d->{uptime} = 0;
2697         $d->{cpu} = 0;
2698         $d->{mem} = 0;
2699
2700         $d->{netout} = 0;
2701         $d->{netin} = 0;
2702
2703         $d->{diskread} = 0;
2704         $d->{diskwrite} = 0;
2705
2706         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2707
2708         $d->{serial} = 1 if conf_has_serial($conf);
2709         $d->{lock} = $conf->{lock} if $conf->{lock};
2710         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2711
2712         $res->{$vmid} = $d;
2713     }
2714
2715     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2716     foreach my $dev (keys %$netdev) {
2717         next if $dev !~ m/^tap([1-9]\d*)i/;
2718         my $vmid = $1;
2719         my $d = $res->{$vmid};
2720         next if !$d;
2721
2722         $d->{netout} += $netdev->{$dev}->{receive};
2723         $d->{netin} += $netdev->{$dev}->{transmit};
2724
2725         if ($full) {
2726             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2727             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2728         }
2729
2730     }
2731
2732     my $ctime = gettimeofday;
2733
2734     foreach my $vmid (keys %$list) {
2735
2736         my $d = $res->{$vmid};
2737         my $pid = $d->{pid};
2738         next if !$pid;
2739
2740         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2741         next if !$pstat; # not running
2742
2743         my $used = $pstat->{utime} + $pstat->{stime};
2744
2745         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2746
2747         if ($pstat->{vsize}) {
2748             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2749         }
2750
2751         my $old = $last_proc_pid_stat->{$pid};
2752         if (!$old) {
2753             $last_proc_pid_stat->{$pid} = {
2754                 time => $ctime,
2755                 used => $used,
2756                 cpu => 0,
2757             };
2758             next;
2759         }
2760
2761         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2762
2763         if ($dtime > 1000) {
2764             my $dutime = $used -  $old->{used};
2765
2766             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2767             $last_proc_pid_stat->{$pid} = {
2768                 time => $ctime,
2769                 used => $used,
2770                 cpu => $d->{cpu},
2771             };
2772         } else {
2773             $d->{cpu} = $old->{cpu};
2774         }
2775     }
2776
2777     return $res if !$full;
2778
2779     my $qmpclient = PVE::QMPClient->new();
2780
2781     my $ballooncb = sub {
2782         my ($vmid, $resp) = @_;
2783
2784         my $info = $resp->{'return'};
2785         return if !$info->{max_mem};
2786
2787         my $d = $res->{$vmid};
2788
2789         # use memory assigned to VM
2790         $d->{maxmem} = $info->{max_mem};
2791         $d->{balloon} = $info->{actual};
2792
2793         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2794             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2795             $d->{freemem} = $info->{free_mem};
2796         }
2797
2798         $d->{ballooninfo} = $info;
2799     };
2800
2801     my $blockstatscb = sub {
2802         my ($vmid, $resp) = @_;
2803         my $data = $resp->{'return'} || [];
2804         my $totalrdbytes = 0;
2805         my $totalwrbytes = 0;
2806
2807         for my $blockstat (@$data) {
2808             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2809             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2810
2811             $blockstat->{device} =~ s/drive-//;
2812             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2813         }
2814         $res->{$vmid}->{diskread} = $totalrdbytes;
2815         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2816     };
2817
2818     my $machinecb = sub {
2819         my ($vmid, $resp) = @_;
2820         my $data = $resp->{'return'} || [];
2821
2822         $res->{$vmid}->{'running-machine'} =
2823             PVE::QemuServer::Machine::current_from_query_machines($data);
2824     };
2825
2826     my $versioncb = sub {
2827         my ($vmid, $resp) = @_;
2828         my $data = $resp->{'return'} // {};
2829         my $version = 'unknown';
2830
2831         if (my $v = $data->{qemu}) {
2832             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2833         }
2834
2835         $res->{$vmid}->{'running-qemu'} = $version;
2836     };
2837
2838     my $statuscb = sub {
2839         my ($vmid, $resp) = @_;
2840
2841         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2842         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2843         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2844         # this fails if ballon driver is not loaded, so this must be
2845         # the last commnand (following command are aborted if this fails).
2846         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2847
2848         my $status = 'unknown';
2849         if (!defined($status = $resp->{'return'}->{status})) {
2850             warn "unable to get VM status\n";
2851             return;
2852         }
2853
2854         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2855     };
2856
2857     foreach my $vmid (keys %$list) {
2858         next if $opt_vmid && ($vmid ne $opt_vmid);
2859         next if !$res->{$vmid}->{pid}; # not running
2860         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2861     }
2862
2863     $qmpclient->queue_execute(undef, 2);
2864
2865     foreach my $vmid (keys %$list) {
2866         next if $opt_vmid && ($vmid ne $opt_vmid);
2867         next if !$res->{$vmid}->{pid}; #not running
2868
2869         # we can't use the $qmpclient since it might have already aborted on
2870         # 'query-balloon', but this might also fail for older versions...
2871         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2872         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2873     }
2874
2875     foreach my $vmid (keys %$list) {
2876         next if $opt_vmid && ($vmid ne $opt_vmid);
2877         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2878     }
2879
2880     return $res;
2881 }
2882
2883 sub conf_has_serial {
2884     my ($conf) = @_;
2885
2886     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2887         if ($conf->{"serial$i"}) {
2888             return 1;
2889         }
2890     }
2891
2892     return 0;
2893 }
2894
2895 sub conf_has_audio {
2896     my ($conf, $id) = @_;
2897
2898     $id //= 0;
2899     my $audio = $conf->{"audio$id"};
2900     return if !defined($audio);
2901
2902     my $audioproperties = parse_property_string($audio_fmt, $audio);
2903     my $audiodriver = $audioproperties->{driver} // 'spice';
2904
2905     return {
2906         dev => $audioproperties->{device},
2907         dev_id => "audiodev$id",
2908         backend => $audiodriver,
2909         backend_id => "$audiodriver-backend${id}",
2910     };
2911 }
2912
2913 sub audio_devs {
2914     my ($audio, $audiopciaddr, $machine_version) = @_;
2915
2916     my $devs = [];
2917
2918     my $id = $audio->{dev_id};
2919     my $audiodev = "";
2920     if (min_version($machine_version, 4, 2)) {
2921         $audiodev = ",audiodev=$audio->{backend_id}";
2922     }
2923
2924     if ($audio->{dev} eq 'AC97') {
2925         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2926     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2927         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2928         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2929         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2930     } else {
2931         die "unkown audio device '$audio->{dev}', implement me!";
2932     }
2933
2934     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2935
2936     return $devs;
2937 }
2938
2939 sub vga_conf_has_spice {
2940     my ($vga) = @_;
2941
2942     my $vgaconf = parse_vga($vga);
2943     my $vgatype = $vgaconf->{type};
2944     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2945
2946     return $1 || 1;
2947 }
2948
2949 sub is_native($) {
2950     my ($arch) = @_;
2951     return get_host_arch() eq $arch;
2952 }
2953
2954 sub get_vm_arch {
2955     my ($conf) = @_;
2956     return $conf->{arch} // get_host_arch();
2957 }
2958
2959 my $default_machines = {
2960     x86_64 => 'pc',
2961     aarch64 => 'virt',
2962 };
2963
2964 sub get_installed_machine_version {
2965     my ($kvmversion) = @_;
2966     $kvmversion = kvm_user_version() if !defined($kvmversion);
2967     $kvmversion =~ m/^(\d+\.\d+)/;
2968     return $1;
2969 }
2970
2971 sub windows_get_pinned_machine_version {
2972     my ($machine, $base_version, $kvmversion) = @_;
2973
2974     my $pin_version = $base_version;
2975     if (!defined($base_version) ||
2976         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2977     ) {
2978         $pin_version = get_installed_machine_version($kvmversion);
2979     }
2980     if (!$machine || $machine eq 'pc') {
2981         $machine = "pc-i440fx-$pin_version";
2982     } elsif ($machine eq 'q35') {
2983         $machine = "pc-q35-$pin_version";
2984     } elsif ($machine eq 'virt') {
2985         $machine = "virt-$pin_version";
2986     } else {
2987         warn "unknown machine type '$machine', not touching that!\n";
2988     }
2989
2990     return $machine;
2991 }
2992
2993 sub get_vm_machine {
2994     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2995
2996     my $machine = $forcemachine || $conf->{machine};
2997
2998     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2999         $kvmversion //= kvm_user_version();
3000         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3001         # layout which confuses windows quite a bit and may result in various regressions..
3002         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3003         if (windows_version($conf->{ostype})) {
3004             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3005         }
3006         $arch //= 'x86_64';
3007         $machine ||= $default_machines->{$arch};
3008         if ($add_pve_version) {
3009             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3010             $machine .= "+pve$pvever";
3011         }
3012     }
3013
3014     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3015         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3016         $machine = $1 if $is_pxe;
3017
3018         # for version-pinned machines that do not include a pve-version (e.g.
3019         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3020         $machine .= '+pve0';
3021
3022         $machine .= '.pxe' if $is_pxe;
3023     }
3024
3025     return $machine;
3026 }
3027
3028 sub get_ovmf_files($) {
3029     my ($arch) = @_;
3030
3031     my $ovmf = $OVMF->{$arch}
3032         or die "no OVMF images known for architecture '$arch'\n";
3033
3034     return @$ovmf;
3035 }
3036
3037 my $Arch2Qemu = {
3038     aarch64 => '/usr/bin/qemu-system-aarch64',
3039     x86_64 => '/usr/bin/qemu-system-x86_64',
3040 };
3041 sub get_command_for_arch($) {
3042     my ($arch) = @_;
3043     return '/usr/bin/kvm' if is_native($arch);
3044
3045     my $cmd = $Arch2Qemu->{$arch}
3046         or die "don't know how to emulate architecture '$arch'\n";
3047     return $cmd;
3048 }
3049
3050 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3051 # to use in a QEMU command line (-cpu element), first array_intersect the result
3052 # of query_supported_ with query_understood_. This is necessary because:
3053 #
3054 # a) query_understood_ returns flags the host cannot use and
3055 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3056 #    flags, but CPU settings - with most of them being flags. Those settings
3057 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3058 #
3059 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3060 # expensive. If you need the value returned from this, you can get it much
3061 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3062 # $accel being 'kvm' or 'tcg'.
3063 #
3064 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3065 # changes, automatically populating pmxcfs.
3066 #
3067 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3068 # since kvm and tcg machines support different flags
3069 #
3070 sub query_supported_cpu_flags {
3071     my ($arch) = @_;
3072
3073     $arch //= get_host_arch();
3074     my $default_machine = $default_machines->{$arch};
3075
3076     my $flags = {};
3077
3078     # FIXME: Once this is merged, the code below should work for ARM as well:
3079     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3080     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3081         $arch eq "aarch64";
3082
3083     my $kvm_supported = defined(kvm_version());
3084     my $qemu_cmd = get_command_for_arch($arch);
3085     my $fakevmid = -1;
3086     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3087
3088     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3089     my $query_supported_run_qemu = sub {
3090         my ($kvm) = @_;
3091
3092         my $flags = {};
3093         my $cmd = [
3094             $qemu_cmd,
3095             '-machine', $default_machine,
3096             '-display', 'none',
3097             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3098             '-mon', 'chardev=qmp,mode=control',
3099             '-pidfile', $pidfile,
3100             '-S', '-daemonize'
3101         ];
3102
3103         if (!$kvm) {
3104             push @$cmd, '-accel', 'tcg';
3105         }
3106
3107         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3108         die "QEMU flag querying VM exited with code " . $rc if $rc;
3109
3110         eval {
3111             my $cmd_result = mon_cmd(
3112                 $fakevmid,
3113                 'query-cpu-model-expansion',
3114                 type => 'full',
3115                 model => { name => 'host' }
3116             );
3117
3118             my $props = $cmd_result->{model}->{props};
3119             foreach my $prop (keys %$props) {
3120                 next if $props->{$prop} ne '1';
3121                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3122                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3123                 # We only keep those with underscores, to match /proc/cpuinfo
3124                 $prop =~ s/\.|-/_/g;
3125                 $flags->{$prop} = 1;
3126             }
3127         };
3128         my $err = $@;
3129
3130         # force stop with 10 sec timeout and 'nocheck'
3131         # always stop, even if QMP failed
3132         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3133
3134         die $err if $err;
3135
3136         return [ sort keys %$flags ];
3137     };
3138
3139     # We need to query QEMU twice, since KVM and TCG have different supported flags
3140     PVE::QemuConfig->lock_config($fakevmid, sub {
3141         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3142         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3143
3144         if ($kvm_supported) {
3145             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3146             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3147         }
3148     });
3149
3150     return $flags;
3151 }
3152
3153 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3154 my $understood_cpu_flag_dir = "/usr/share/kvm";
3155 sub query_understood_cpu_flags {
3156     my $arch = get_host_arch();
3157     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3158
3159     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3160         if ! -e $filepath;
3161
3162     my $raw = file_get_contents($filepath);
3163     $raw =~ s/^\s+|\s+$//g;
3164     my @flags = split(/\s+/, $raw);
3165
3166     return \@flags;
3167 }
3168
3169 sub config_to_command {
3170     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3171         $pbs_backing) = @_;
3172
3173     my $cmd = [];
3174     my $globalFlags = [];
3175     my $machineFlags = [];
3176     my $rtcFlags = [];
3177     my $devices = [];
3178     my $pciaddr = '';
3179     my $bridges = {};
3180     my $ostype = $conf->{ostype};
3181     my $winversion = windows_version($ostype);
3182     my $kvm = $conf->{kvm};
3183     my $nodename = nodename();
3184
3185     my $arch = get_vm_arch($conf);
3186     my $kvm_binary = get_command_for_arch($arch);
3187     my $kvmver = kvm_user_version($kvm_binary);
3188
3189     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3190         $kvmver //= "undefined";
3191         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3192     }
3193
3194     my $add_pve_version = min_version($kvmver, 4, 1);
3195
3196     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3197     my $machine_version = extract_version($machine_type, $kvmver);
3198     $kvm //= 1 if is_native($arch);
3199
3200     $machine_version =~ m/(\d+)\.(\d+)/;
3201     my ($machine_major, $machine_minor) = ($1, $2);
3202
3203     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3204         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3205     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3206         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3207             ." please upgrade node '$nodename'\n"
3208     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3209         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3210         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3211             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3212             ." node '$nodename'\n";
3213     }
3214
3215     # if a specific +pve version is required for a feature, use $version_guard
3216     # instead of min_version to allow machines to be run with the minimum
3217     # required version
3218     my $required_pve_version = 0;
3219     my $version_guard = sub {
3220         my ($major, $minor, $pve) = @_;
3221         return 0 if !min_version($machine_version, $major, $minor, $pve);
3222         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3223         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3224         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3225         return 1;
3226     };
3227
3228     if ($kvm && !defined kvm_version()) {
3229         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3230             ." or enable in BIOS.\n";
3231     }
3232
3233     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3234     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3235     my $use_old_bios_files = undef;
3236     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3237
3238     my $cpuunits = defined($conf->{cpuunits}) ?
3239             $conf->{cpuunits} : $defaults->{cpuunits};
3240
3241     push @$cmd, $kvm_binary;
3242
3243     push @$cmd, '-id', $vmid;
3244
3245     my $vmname = $conf->{name} || "vm$vmid";
3246
3247     push @$cmd, '-name', $vmname;
3248
3249     push @$cmd, '-no-shutdown';
3250
3251     my $use_virtio = 0;
3252
3253     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3254     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3255     push @$cmd, '-mon', "chardev=qmp,mode=control";
3256
3257     if (min_version($machine_version, 2, 12)) {
3258         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3259         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3260     }
3261
3262     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3263
3264     push @$cmd, '-daemonize';
3265
3266     if ($conf->{smbios1}) {
3267         my $smbios_conf = parse_smbios1($conf->{smbios1});
3268         if ($smbios_conf->{base64}) {
3269             # Do not pass base64 flag to qemu
3270             delete $smbios_conf->{base64};
3271             my $smbios_string = "";
3272             foreach my $key (keys %$smbios_conf) {
3273                 my $value;
3274                 if ($key eq "uuid") {
3275                     $value = $smbios_conf->{uuid}
3276                 } else {
3277                     $value = decode_base64($smbios_conf->{$key});
3278                 }
3279                 # qemu accepts any binary data, only commas need escaping by double comma
3280                 $value =~ s/,/,,/g;
3281                 $smbios_string .= "," . $key . "=" . $value if $value;
3282             }
3283             push @$cmd, '-smbios', "type=1" . $smbios_string;
3284         } else {
3285             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3286         }
3287     }
3288
3289     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3290         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3291         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3292
3293         my ($path, $format);
3294         if (my $efidisk = $conf->{efidisk0}) {
3295             my $d = parse_drive('efidisk0', $efidisk);
3296             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3297             $format = $d->{format};
3298             if ($storeid) {
3299                 $path = PVE::Storage::path($storecfg, $d->{file});
3300                 if (!defined($format)) {
3301                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3302                     $format = qemu_img_format($scfg, $volname);
3303                 }
3304             } else {
3305                 $path = $d->{file};
3306                 die "efidisk format must be specified\n"
3307                     if !defined($format);
3308             }
3309         } else {
3310             warn "no efidisk configured! Using temporary efivars disk.\n";
3311             $path = "/tmp/$vmid-ovmf.fd";
3312             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3313             $format = 'raw';
3314         }
3315
3316         my $size_str = "";
3317
3318         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3319             $size_str = ",size=" . (-s $ovmf_vars);
3320         }
3321
3322         # on slower ceph clusters, booting without cache on efidisk can take a while, see #3329
3323         my $cache = $path =~ m/^rbd:/ ? ',cache=writeback' : '';
3324
3325         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3326         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=$path";
3327     }
3328
3329     # load q35 config
3330     if ($q35) {
3331         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3332         if (min_version($machine_version, 4, 0)) {
3333             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3334         } else {
3335             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3336         }
3337     }
3338
3339     if ($conf->{vmgenid}) {
3340         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3341     }
3342
3343     # add usb controllers
3344     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3345         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3346     push @$devices, @usbcontrollers if @usbcontrollers;
3347     my $vga = parse_vga($conf->{vga});
3348
3349     my $qxlnum = vga_conf_has_spice($conf->{vga});
3350     $vga->{type} = 'qxl' if $qxlnum;
3351
3352     if (!$vga->{type}) {
3353         if ($arch eq 'aarch64') {
3354             $vga->{type} = 'virtio';
3355         } elsif (min_version($machine_version, 2, 9)) {
3356             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3357         } else {
3358             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3359         }
3360     }
3361
3362     # enable absolute mouse coordinates (needed by vnc)
3363     my $tablet;
3364     if (defined($conf->{tablet})) {
3365         $tablet = $conf->{tablet};
3366     } else {
3367         $tablet = $defaults->{tablet};
3368         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3369         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3370     }
3371
3372     if ($tablet) {
3373         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3374         my $kbd = print_keyboarddevice_full($conf, $arch);
3375         push @$devices, '-device', $kbd if defined($kbd);
3376     }
3377
3378     my $bootorder = device_bootorder($conf);
3379
3380     # host pci device passthrough
3381     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3382         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3383
3384     # usb devices
3385     my $usb_dev_features = {};
3386     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3387
3388     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3389         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3390     push @$devices, @usbdevices if @usbdevices;
3391
3392     # serial devices
3393     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3394         if (my $path = $conf->{"serial$i"}) {
3395             if ($path eq 'socket') {
3396                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3397                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3398                 # On aarch64, serial0 is the UART device. Qemu only allows
3399                 # connecting UART devices via the '-serial' command line, as
3400                 # the device has a fixed slot on the hardware...
3401                 if ($arch eq 'aarch64' && $i == 0) {
3402                     push @$devices, '-serial', "chardev:serial$i";
3403                 } else {
3404                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3405                 }
3406             } else {
3407                 die "no such serial device\n" if ! -c $path;
3408                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3409                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3410             }
3411         }
3412     }
3413
3414     # parallel devices
3415     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3416         if (my $path = $conf->{"parallel$i"}) {
3417             die "no such parallel device\n" if ! -c $path;
3418             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3419             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3420             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3421         }
3422     }
3423
3424     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3425         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3426         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3427         push @$devices, @$audio_devs;
3428     }
3429
3430     my $sockets = 1;
3431     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3432     $sockets = $conf->{sockets} if  $conf->{sockets};
3433
3434     my $cores = $conf->{cores} || 1;
3435
3436     my $maxcpus = $sockets * $cores;
3437
3438     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3439
3440     my $allowed_vcpus = $cpuinfo->{cpus};
3441
3442     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3443         if ($allowed_vcpus < $maxcpus);
3444
3445     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3446
3447         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3448         for (my $i = 2; $i <= $vcpus; $i++)  {
3449             my $cpustr = print_cpu_device($conf,$i);
3450             push @$cmd, '-device', $cpustr;
3451         }
3452
3453     } else {
3454
3455         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3456     }
3457     push @$cmd, '-nodefaults';
3458
3459     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3460
3461     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3462
3463     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3464
3465     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3466         push @$devices, '-device', print_vga_device(
3467             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3468         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3469         push @$cmd,  '-vnc', "unix:$socket,password=on";
3470     } else {
3471         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3472         push @$cmd, '-nographic';
3473     }
3474
3475     # time drift fix
3476     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3477     my $useLocaltime = $conf->{localtime};
3478
3479     if ($winversion >= 5) { # windows
3480         $useLocaltime = 1 if !defined($conf->{localtime});
3481
3482         # use time drift fix when acpi is enabled
3483         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3484             $tdf = 1 if !defined($conf->{tdf});
3485         }
3486     }
3487
3488     if ($winversion >= 6) {
3489         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3490         push @$cmd, '-no-hpet';
3491     }
3492
3493     push @$rtcFlags, 'driftfix=slew' if $tdf;
3494
3495     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3496         push @$rtcFlags, "base=$conf->{startdate}";
3497     } elsif ($useLocaltime) {
3498         push @$rtcFlags, 'base=localtime';
3499     }
3500
3501     if ($forcecpu) {
3502         push @$cmd, '-cpu', $forcecpu;
3503     } else {
3504         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3505     }
3506
3507     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3508
3509     push @$cmd, '-S' if $conf->{freeze};
3510
3511     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3512
3513     my $guest_agent = parse_guest_agent($conf);
3514
3515     if ($guest_agent->{enabled}) {
3516         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3517         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3518
3519         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3520             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3521             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3522             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3523         } elsif ($guest_agent->{type} eq 'isa') {
3524             push @$devices, '-device', "isa-serial,chardev=qga0";
3525         }
3526     }
3527
3528     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3529     if ($rng && $version_guard->(4, 1, 2)) {
3530         check_rng_source($rng->{source});
3531
3532         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3533         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3534         my $limiter_str = "";
3535         if ($max_bytes) {
3536             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3537         }
3538
3539         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3540         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3541         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3542     }
3543
3544     my $spice_port;
3545
3546     if ($qxlnum) {
3547         if ($qxlnum > 1) {
3548             if ($winversion){
3549                 for (my $i = 1; $i < $qxlnum; $i++){
3550                     push @$devices, '-device', print_vga_device(
3551                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3552                 }
3553             } else {
3554                 # assume other OS works like Linux
3555                 my ($ram, $vram) = ("134217728", "67108864");
3556                 if ($vga->{memory}) {
3557                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3558                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3559                 }
3560                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3561                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3562             }
3563         }
3564
3565         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3566
3567         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3568         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3569         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3570
3571         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3572         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3573         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3574
3575         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3576         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3577
3578         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3579         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3580         if ($spice_enhancement->{foldersharing}) {
3581             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3582             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3583         }
3584
3585         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3586         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3587             if $spice_enhancement->{videostreaming};
3588
3589         push @$devices, '-spice', "$spice_opts";
3590     }
3591
3592     # enable balloon by default, unless explicitly disabled
3593     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3594         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3595         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3596     }
3597
3598     if ($conf->{watchdog}) {
3599         my $wdopts = parse_watchdog($conf->{watchdog});
3600         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3601         my $watchdog = $wdopts->{model} || 'i6300esb';
3602         push @$devices, '-device', "$watchdog$pciaddr";
3603         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3604     }
3605
3606     my $vollist = [];
3607     my $scsicontroller = {};
3608     my $ahcicontroller = {};
3609     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3610
3611     # Add iscsi initiator name if available
3612     if (my $initiator = get_initiator_name()) {
3613         push @$devices, '-iscsi', "initiator-name=$initiator";
3614     }
3615
3616     PVE::QemuConfig->foreach_volume($conf, sub {
3617         my ($ds, $drive) = @_;
3618
3619         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3620             check_volume_storage_type($storecfg, $drive->{file});
3621             push @$vollist, $drive->{file};
3622         }
3623
3624         # ignore efidisk here, already added in bios/fw handling code above
3625         return if $drive->{interface} eq 'efidisk';
3626
3627         $use_virtio = 1 if $ds =~ m/^virtio/;
3628
3629         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3630
3631         if ($drive->{interface} eq 'virtio'){
3632            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3633         }
3634
3635         if ($drive->{interface} eq 'scsi') {
3636
3637             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3638
3639             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3640                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3641
3642             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3643             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3644
3645             my $iothread = '';
3646             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3647                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3648                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3649             } elsif ($drive->{iothread}) {
3650                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3651             }
3652
3653             my $queues = '';
3654             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3655                 $queues = ",num_queues=$drive->{queues}";
3656             }
3657
3658             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3659                 if !$scsicontroller->{$controller};
3660             $scsicontroller->{$controller}=1;
3661         }
3662
3663         if ($drive->{interface} eq 'sata') {
3664             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3665             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3666             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3667                 if !$ahcicontroller->{$controller};
3668             $ahcicontroller->{$controller}=1;
3669         }
3670
3671         my $pbs_conf = $pbs_backing->{$ds};
3672         my $pbs_name = undef;
3673         if ($pbs_conf) {
3674             $pbs_name = "drive-$ds-pbs";
3675             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3676         }
3677
3678         my $drive_cmd = print_drive_commandline_full(
3679             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3680
3681         # extra protection for templates, but SATA and IDE don't support it..
3682         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3683
3684         push @$devices, '-drive',$drive_cmd;
3685         push @$devices, '-device', print_drivedevice_full(
3686             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3687     });
3688
3689     for (my $i = 0; $i < $MAX_NETS; $i++) {
3690         my $netname = "net$i";
3691
3692         next if !$conf->{$netname};
3693         my $d = parse_net($conf->{$netname});
3694         next if !$d;
3695
3696         $use_virtio = 1 if $d->{model} eq 'virtio';
3697
3698         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3699
3700         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3701         push @$devices, '-netdev', $netdevfull;
3702
3703         my $netdevicefull = print_netdevice_full(
3704             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3705
3706         push @$devices, '-device', $netdevicefull;
3707     }
3708
3709     if ($conf->{ivshmem}) {
3710         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3711
3712         my $bus;
3713         if ($q35) {
3714             $bus = print_pcie_addr("ivshmem");
3715         } else {
3716             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3717         }
3718
3719         my $ivshmem_name = $ivshmem->{name} // $vmid;
3720         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3721
3722         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3723         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3724             .",size=$ivshmem->{size}M";
3725     }
3726
3727     # pci.4 is nested in pci.1
3728     $bridges->{1} = 1 if $bridges->{4};
3729
3730     if (!$q35) {
3731         # add pci bridges
3732         if (min_version($machine_version, 2, 3)) {
3733            $bridges->{1} = 1;
3734            $bridges->{2} = 1;
3735         }
3736
3737         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3738
3739     }
3740
3741     for my $k (sort {$b cmp $a} keys %$bridges) {
3742         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3743
3744         my $k_name = $k;
3745         if ($k == 2 && $legacy_igd) {
3746             $k_name = "$k-igd";
3747         }
3748         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3749
3750         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3751         if ($q35) {
3752             # add after -readconfig pve-q35.cfg
3753             splice @$devices, 2, 0, '-device', $devstr;
3754         } else {
3755             unshift @$devices, '-device', $devstr if $k > 0;
3756         }
3757     }
3758
3759     if (!$kvm) {
3760         push @$machineFlags, 'accel=tcg';
3761     }
3762
3763     my $machine_type_min = $machine_type;
3764     if ($add_pve_version) {
3765         $machine_type_min =~ s/\+pve\d+$//;
3766         $machine_type_min .= "+pve$required_pve_version";
3767     }
3768     push @$machineFlags, "type=${machine_type_min}";
3769
3770     push @$cmd, @$devices;
3771     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3772     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3773     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3774
3775     if (my $vmstate = $conf->{vmstate}) {
3776         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3777         push @$vollist, $vmstate;
3778         push @$cmd, '-loadstate', $statepath;
3779         print "activating and using '$vmstate' as vmstate\n";
3780     }
3781
3782     # add custom args
3783     if ($conf->{args}) {
3784         my $aa = PVE::Tools::split_args($conf->{args});
3785         push @$cmd, @$aa;
3786     }
3787
3788     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3789 }
3790
3791 sub check_rng_source {
3792     my ($source) = @_;
3793
3794     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3795     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3796         if ! -e $source;
3797
3798     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3799     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3800         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3801         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3802         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3803             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3804             ." to the host.\n";
3805     }
3806 }
3807
3808 sub spice_port {
3809     my ($vmid) = @_;
3810
3811     my $res = mon_cmd($vmid, 'query-spice');
3812
3813     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3814 }
3815
3816 sub vm_devices_list {
3817     my ($vmid) = @_;
3818
3819     my $res = mon_cmd($vmid, 'query-pci');
3820     my $devices_to_check = [];
3821     my $devices = {};
3822     foreach my $pcibus (@$res) {
3823         push @$devices_to_check, @{$pcibus->{devices}},
3824     }
3825
3826     while (@$devices_to_check) {
3827         my $to_check = [];
3828         for my $d (@$devices_to_check) {
3829             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3830             next if !$d->{'pci_bridge'};
3831
3832             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3833             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3834         }
3835         $devices_to_check = $to_check;
3836     }
3837
3838     my $resblock = mon_cmd($vmid, 'query-block');
3839     foreach my $block (@$resblock) {
3840         if($block->{device} =~ m/^drive-(\S+)/){
3841                 $devices->{$1} = 1;
3842         }
3843     }
3844
3845     my $resmice = mon_cmd($vmid, 'query-mice');
3846     foreach my $mice (@$resmice) {
3847         if ($mice->{name} eq 'QEMU HID Tablet') {
3848             $devices->{tablet} = 1;
3849             last;
3850         }
3851     }
3852
3853     # for usb devices there is no query-usb
3854     # but we can iterate over the entries in
3855     # qom-list path=/machine/peripheral
3856     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3857     foreach my $per (@$resperipheral) {
3858         if ($per->{name} =~ m/^usb\d+$/) {
3859             $devices->{$per->{name}} = 1;
3860         }
3861     }
3862
3863     return $devices;
3864 }
3865
3866 sub vm_deviceplug {
3867     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3868
3869     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3870
3871     my $devices_list = vm_devices_list($vmid);
3872     return 1 if defined($devices_list->{$deviceid});
3873
3874     # add PCI bridge if we need it for the device
3875     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3876
3877     if ($deviceid eq 'tablet') {
3878
3879         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3880
3881     } elsif ($deviceid eq 'keyboard') {
3882
3883         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3884
3885     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3886
3887         die "usb hotplug currently not reliable\n";
3888         # since we can't reliably hot unplug all added usb devices and usb
3889         # passthrough breaks live migration we disable usb hotplugging for now
3890         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3891
3892     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3893
3894         qemu_iothread_add($vmid, $deviceid, $device);
3895
3896         qemu_driveadd($storecfg, $vmid, $device);
3897         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3898
3899         qemu_deviceadd($vmid, $devicefull);
3900         eval { qemu_deviceaddverify($vmid, $deviceid); };
3901         if (my $err = $@) {
3902             eval { qemu_drivedel($vmid, $deviceid); };
3903             warn $@ if $@;
3904             die $err;
3905         }
3906
3907     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3908
3909
3910         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3911         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3912         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3913
3914         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3915
3916         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3917             qemu_iothread_add($vmid, $deviceid, $device);
3918             $devicefull .= ",iothread=iothread-$deviceid";
3919         }
3920
3921         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3922             $devicefull .= ",num_queues=$device->{queues}";
3923         }
3924
3925         qemu_deviceadd($vmid, $devicefull);
3926         qemu_deviceaddverify($vmid, $deviceid);
3927
3928     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3929
3930         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3931         qemu_driveadd($storecfg, $vmid, $device);
3932
3933         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3934         eval { qemu_deviceadd($vmid, $devicefull); };
3935         if (my $err = $@) {
3936             eval { qemu_drivedel($vmid, $deviceid); };
3937             warn $@ if $@;
3938             die $err;
3939         }
3940
3941     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3942
3943         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3944
3945         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3946         my $use_old_bios_files = undef;
3947         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3948
3949         my $netdevicefull = print_netdevice_full(
3950             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3951         qemu_deviceadd($vmid, $netdevicefull);
3952         eval {
3953             qemu_deviceaddverify($vmid, $deviceid);
3954             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3955         };
3956         if (my $err = $@) {
3957             eval { qemu_netdevdel($vmid, $deviceid); };
3958             warn $@ if $@;
3959             die $err;
3960         }
3961
3962     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3963
3964         my $bridgeid = $2;
3965         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3966         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3967
3968         qemu_deviceadd($vmid, $devicefull);
3969         qemu_deviceaddverify($vmid, $deviceid);
3970
3971     } else {
3972         die "can't hotplug device '$deviceid'\n";
3973     }
3974
3975     return 1;
3976 }
3977
3978 # fixme: this should raise exceptions on error!
3979 sub vm_deviceunplug {
3980     my ($vmid, $conf, $deviceid) = @_;
3981
3982     my $devices_list = vm_devices_list($vmid);
3983     return 1 if !defined($devices_list->{$deviceid});
3984
3985     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3986     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3987
3988     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3989
3990         qemu_devicedel($vmid, $deviceid);
3991
3992     } elsif ($deviceid =~ m/^usb\d+$/) {
3993
3994         die "usb hotplug currently not reliable\n";
3995         # when unplugging usb devices this way, there may be remaining usb
3996         # controllers/hubs so we disable it for now
3997         #qemu_devicedel($vmid, $deviceid);
3998         #qemu_devicedelverify($vmid, $deviceid);
3999
4000     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4001
4002         qemu_devicedel($vmid, $deviceid);
4003         qemu_devicedelverify($vmid, $deviceid);
4004         qemu_drivedel($vmid, $deviceid);
4005         qemu_iothread_del($conf, $vmid, $deviceid);
4006
4007     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4008
4009         qemu_devicedel($vmid, $deviceid);
4010         qemu_devicedelverify($vmid, $deviceid);
4011         qemu_iothread_del($conf, $vmid, $deviceid);
4012
4013     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4014
4015         qemu_devicedel($vmid, $deviceid);
4016         qemu_drivedel($vmid, $deviceid);
4017         qemu_deletescsihw($conf, $vmid, $deviceid);
4018
4019     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4020
4021         qemu_devicedel($vmid, $deviceid);
4022         qemu_devicedelverify($vmid, $deviceid);
4023         qemu_netdevdel($vmid, $deviceid);
4024
4025     } else {
4026         die "can't unplug device '$deviceid'\n";
4027     }
4028
4029     return 1;
4030 }
4031
4032 sub qemu_deviceadd {
4033     my ($vmid, $devicefull) = @_;
4034
4035     $devicefull = "driver=".$devicefull;
4036     my %options =  split(/[=,]/, $devicefull);
4037
4038     mon_cmd($vmid, "device_add" , %options);
4039 }
4040
4041 sub qemu_devicedel {
4042     my ($vmid, $deviceid) = @_;
4043
4044     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4045 }
4046
4047 sub qemu_iothread_add {
4048     my($vmid, $deviceid, $device) = @_;
4049
4050     if ($device->{iothread}) {
4051         my $iothreads = vm_iothreads_list($vmid);
4052         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4053     }
4054 }
4055
4056 sub qemu_iothread_del {
4057     my($conf, $vmid, $deviceid) = @_;
4058
4059     my $confid = $deviceid;
4060     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4061         $confid = 'scsi' . $1;
4062     }
4063     my $device = parse_drive($confid, $conf->{$confid});
4064     if ($device->{iothread}) {
4065         my $iothreads = vm_iothreads_list($vmid);
4066         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4067     }
4068 }
4069
4070 sub qemu_objectadd {
4071     my($vmid, $objectid, $qomtype) = @_;
4072
4073     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4074
4075     return 1;
4076 }
4077
4078 sub qemu_objectdel {
4079     my($vmid, $objectid) = @_;
4080
4081     mon_cmd($vmid, "object-del", id => $objectid);
4082
4083     return 1;
4084 }
4085
4086 sub qemu_driveadd {
4087     my ($storecfg, $vmid, $device) = @_;
4088
4089     my $kvmver = get_running_qemu_version($vmid);
4090     my $io_uring = min_version($kvmver, 6, 0);
4091     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4092     $drive =~ s/\\/\\\\/g;
4093     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4094
4095     # If the command succeeds qemu prints: "OK"
4096     return 1 if $ret =~ m/OK/s;
4097
4098     die "adding drive failed: $ret\n";
4099 }
4100
4101 sub qemu_drivedel {
4102     my($vmid, $deviceid) = @_;
4103
4104     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4105     $ret =~ s/^\s+//;
4106
4107     return 1 if $ret eq "";
4108
4109     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4110     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4111
4112     die "deleting drive $deviceid failed : $ret\n";
4113 }
4114
4115 sub qemu_deviceaddverify {
4116     my ($vmid, $deviceid) = @_;
4117
4118     for (my $i = 0; $i <= 5; $i++) {
4119          my $devices_list = vm_devices_list($vmid);
4120          return 1 if defined($devices_list->{$deviceid});
4121          sleep 1;
4122     }
4123
4124     die "error on hotplug device '$deviceid'\n";
4125 }
4126
4127
4128 sub qemu_devicedelverify {
4129     my ($vmid, $deviceid) = @_;
4130
4131     # need to verify that the device is correctly removed as device_del
4132     # is async and empty return is not reliable
4133
4134     for (my $i = 0; $i <= 5; $i++) {
4135          my $devices_list = vm_devices_list($vmid);
4136          return 1 if !defined($devices_list->{$deviceid});
4137          sleep 1;
4138     }
4139
4140     die "error on hot-unplugging device '$deviceid'\n";
4141 }
4142
4143 sub qemu_findorcreatescsihw {
4144     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4145
4146     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4147
4148     my $scsihwid="$controller_prefix$controller";
4149     my $devices_list = vm_devices_list($vmid);
4150
4151     if(!defined($devices_list->{$scsihwid})) {
4152         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4153     }
4154
4155     return 1;
4156 }
4157
4158 sub qemu_deletescsihw {
4159     my ($conf, $vmid, $opt) = @_;
4160
4161     my $device = parse_drive($opt, $conf->{$opt});
4162
4163     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4164         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4165         return 1;
4166     }
4167
4168     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4169
4170     my $devices_list = vm_devices_list($vmid);
4171     foreach my $opt (keys %{$devices_list}) {
4172         if (is_valid_drivename($opt)) {
4173             my $drive = parse_drive($opt, $conf->{$opt});
4174             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4175                 return 1;
4176             }
4177         }
4178     }
4179
4180     my $scsihwid="scsihw$controller";
4181
4182     vm_deviceunplug($vmid, $conf, $scsihwid);
4183
4184     return 1;
4185 }
4186
4187 sub qemu_add_pci_bridge {
4188     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4189
4190     my $bridges = {};
4191
4192     my $bridgeid;
4193
4194     print_pci_addr($device, $bridges, $arch, $machine_type);
4195
4196     while (my ($k, $v) = each %$bridges) {
4197         $bridgeid = $k;
4198     }
4199     return 1 if !defined($bridgeid) || $bridgeid < 1;
4200
4201     my $bridge = "pci.$bridgeid";
4202     my $devices_list = vm_devices_list($vmid);
4203
4204     if (!defined($devices_list->{$bridge})) {
4205         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4206     }
4207
4208     return 1;
4209 }
4210
4211 sub qemu_set_link_status {
4212     my ($vmid, $device, $up) = @_;
4213
4214     mon_cmd($vmid, "set_link", name => $device,
4215                up => $up ? JSON::true : JSON::false);
4216 }
4217
4218 sub qemu_netdevadd {
4219     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4220
4221     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4222     my %options =  split(/[=,]/, $netdev);
4223
4224     if (defined(my $vhost = $options{vhost})) {
4225         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4226     }
4227
4228     if (defined(my $queues = $options{queues})) {
4229         $options{queues} = $queues + 0;
4230     }
4231
4232     mon_cmd($vmid, "netdev_add",  %options);
4233     return 1;
4234 }
4235
4236 sub qemu_netdevdel {
4237     my ($vmid, $deviceid) = @_;
4238
4239     mon_cmd($vmid, "netdev_del", id => $deviceid);
4240 }
4241
4242 sub qemu_usb_hotplug {
4243     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4244
4245     return if !$device;
4246
4247     # remove the old one first
4248     vm_deviceunplug($vmid, $conf, $deviceid);
4249
4250     # check if xhci controller is necessary and available
4251     if ($device->{usb3}) {
4252
4253         my $devicelist = vm_devices_list($vmid);
4254
4255         if (!$devicelist->{xhci}) {
4256             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4257             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4258         }
4259     }
4260     my $d = parse_usb_device($device->{host});
4261     $d->{usb3} = $device->{usb3};
4262
4263     # add the new one
4264     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4265 }
4266
4267 sub qemu_cpu_hotplug {
4268     my ($vmid, $conf, $vcpus) = @_;
4269
4270     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4271
4272     my $sockets = 1;
4273     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4274     $sockets = $conf->{sockets} if  $conf->{sockets};
4275     my $cores = $conf->{cores} || 1;
4276     my $maxcpus = $sockets * $cores;
4277
4278     $vcpus = $maxcpus if !$vcpus;
4279
4280     die "you can't add more vcpus than maxcpus\n"
4281         if $vcpus > $maxcpus;
4282
4283     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4284
4285     if ($vcpus < $currentvcpus) {
4286
4287         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4288
4289             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4290                 qemu_devicedel($vmid, "cpu$i");
4291                 my $retry = 0;
4292                 my $currentrunningvcpus = undef;
4293                 while (1) {
4294                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4295                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4296                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4297                     $retry++;
4298                     sleep 1;
4299                 }
4300                 #update conf after each succesfull cpu unplug
4301                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4302                 PVE::QemuConfig->write_config($vmid, $conf);
4303             }
4304         } else {
4305             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4306         }
4307
4308         return;
4309     }
4310
4311     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4312     die "vcpus in running vm does not match its configuration\n"
4313         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4314
4315     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4316
4317         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4318             my $cpustr = print_cpu_device($conf, $i);
4319             qemu_deviceadd($vmid, $cpustr);
4320
4321             my $retry = 0;
4322             my $currentrunningvcpus = undef;
4323             while (1) {
4324                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4325                 last if scalar(@{$currentrunningvcpus}) == $i;
4326                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4327                 sleep 1;
4328                 $retry++;
4329             }
4330             #update conf after each succesfull cpu hotplug
4331             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4332             PVE::QemuConfig->write_config($vmid, $conf);
4333         }
4334     } else {
4335
4336         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4337             mon_cmd($vmid, "cpu-add", id => int($i));
4338         }
4339     }
4340 }
4341
4342 sub qemu_block_set_io_throttle {
4343     my ($vmid, $deviceid,
4344         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4345         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4346         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4347         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4348
4349     return if !check_running($vmid) ;
4350
4351     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4352         bps => int($bps),
4353         bps_rd => int($bps_rd),
4354         bps_wr => int($bps_wr),
4355         iops => int($iops),
4356         iops_rd => int($iops_rd),
4357         iops_wr => int($iops_wr),
4358         bps_max => int($bps_max),
4359         bps_rd_max => int($bps_rd_max),
4360         bps_wr_max => int($bps_wr_max),
4361         iops_max => int($iops_max),
4362         iops_rd_max => int($iops_rd_max),
4363         iops_wr_max => int($iops_wr_max),
4364         bps_max_length => int($bps_max_length),
4365         bps_rd_max_length => int($bps_rd_max_length),
4366         bps_wr_max_length => int($bps_wr_max_length),
4367         iops_max_length => int($iops_max_length),
4368         iops_rd_max_length => int($iops_rd_max_length),
4369         iops_wr_max_length => int($iops_wr_max_length),
4370     );
4371
4372 }
4373
4374 sub qemu_block_resize {
4375     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4376
4377     my $running = check_running($vmid);
4378
4379     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4380
4381     return if !$running;
4382
4383     my $padding = (1024 - $size % 1024) % 1024;
4384     $size = $size + $padding;
4385
4386     mon_cmd(
4387         $vmid,
4388         "block_resize",
4389         device => $deviceid,
4390         size => int($size),
4391         timeout => 60,
4392     );
4393 }
4394
4395 sub qemu_volume_snapshot {
4396     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4397
4398     my $running = check_running($vmid);
4399
4400     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4401         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4402     } else {
4403         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4404     }
4405 }
4406
4407 sub qemu_volume_snapshot_delete {
4408     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4409
4410     my $running = check_running($vmid);
4411
4412     if($running) {
4413
4414         $running = undef;
4415         my $conf = PVE::QemuConfig->load_config($vmid);
4416         PVE::QemuConfig->foreach_volume($conf, sub {
4417             my ($ds, $drive) = @_;
4418             $running = 1 if $drive->{file} eq $volid;
4419         });
4420     }
4421
4422     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4423         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4424     } else {
4425         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4426     }
4427 }
4428
4429 sub set_migration_caps {
4430     my ($vmid, $savevm) = @_;
4431
4432     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4433
4434     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4435     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4436
4437     my $cap_ref = [];
4438
4439     my $enabled_cap = {
4440         "auto-converge" => 1,
4441         "xbzrle" => 1,
4442         "x-rdma-pin-all" => 0,
4443         "zero-blocks" => 0,
4444         "compress" => 0,
4445         "dirty-bitmaps" => $dirty_bitmaps,
4446     };
4447
4448     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4449
4450     for my $supported_capability (@$supported_capabilities) {
4451         push @$cap_ref, {
4452             capability => $supported_capability->{capability},
4453             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4454         };
4455     }
4456
4457     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4458 }
4459
4460 sub foreach_volid {
4461     my ($conf, $func, @param) = @_;
4462
4463     my $volhash = {};
4464
4465     my $test_volid = sub {
4466         my ($key, $drive, $snapname) = @_;
4467
4468         my $volid = $drive->{file};
4469         return if !$volid;
4470
4471         $volhash->{$volid}->{cdrom} //= 1;
4472         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4473
4474         my $replicate = $drive->{replicate} // 1;
4475         $volhash->{$volid}->{replicate} //= 0;
4476         $volhash->{$volid}->{replicate} = 1 if $replicate;
4477
4478         $volhash->{$volid}->{shared} //= 0;
4479         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4480
4481         $volhash->{$volid}->{referenced_in_config} //= 0;
4482         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4483
4484         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4485             if defined($snapname);
4486
4487         my $size = $drive->{size};
4488         $volhash->{$volid}->{size} //= $size if $size;
4489
4490         $volhash->{$volid}->{is_vmstate} //= 0;
4491         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4492
4493         $volhash->{$volid}->{is_unused} //= 0;
4494         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4495
4496         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4497     };
4498
4499     my $include_opts = {
4500         extra_keys => ['vmstate'],
4501         include_unused => 1,
4502     };
4503
4504     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4505     foreach my $snapname (keys %{$conf->{snapshots}}) {
4506         my $snap = $conf->{snapshots}->{$snapname};
4507         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4508     }
4509
4510     foreach my $volid (keys %$volhash) {
4511         &$func($volid, $volhash->{$volid}, @param);
4512     }
4513 }
4514
4515 my $fast_plug_option = {
4516     'lock' => 1,
4517     'name' => 1,
4518     'onboot' => 1,
4519     'shares' => 1,
4520     'startup' => 1,
4521     'description' => 1,
4522     'protection' => 1,
4523     'vmstatestorage' => 1,
4524     'hookscript' => 1,
4525     'tags' => 1,
4526 };
4527
4528 # hotplug changes in [PENDING]
4529 # $selection hash can be used to only apply specified options, for
4530 # example: { cores => 1 } (only apply changed 'cores')
4531 # $errors ref is used to return error messages
4532 sub vmconfig_hotplug_pending {
4533     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4534
4535     my $defaults = load_defaults();
4536     my $arch = get_vm_arch($conf);
4537     my $machine_type = get_vm_machine($conf, undef, $arch);
4538
4539     # commit values which do not have any impact on running VM first
4540     # Note: those option cannot raise errors, we we do not care about
4541     # $selection and always apply them.
4542
4543     my $add_error = sub {
4544         my ($opt, $msg) = @_;
4545         $errors->{$opt} = "hotplug problem - $msg";
4546     };
4547
4548     my $changes = 0;
4549     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4550         if ($fast_plug_option->{$opt}) {
4551             $conf->{$opt} = $conf->{pending}->{$opt};
4552             delete $conf->{pending}->{$opt};
4553             $changes = 1;
4554         }
4555     }
4556
4557     if ($changes) {
4558         PVE::QemuConfig->write_config($vmid, $conf);
4559     }
4560
4561     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4562
4563     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4564     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4565     foreach my $opt (sort keys %$pending_delete_hash) {
4566         next if $selection && !$selection->{$opt};
4567         my $force = $pending_delete_hash->{$opt}->{force};
4568         eval {
4569             if ($opt eq 'hotplug') {
4570                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4571             } elsif ($opt eq 'tablet') {
4572                 die "skip\n" if !$hotplug_features->{usb};
4573                 if ($defaults->{tablet}) {
4574                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4575                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4576                         if $arch eq 'aarch64';
4577                 } else {
4578                     vm_deviceunplug($vmid, $conf, 'tablet');
4579                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4580                 }
4581             } elsif ($opt =~ m/^usb\d+/) {
4582                 die "skip\n";
4583                 # since we cannot reliably hot unplug usb devices we are disabling it
4584                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4585                 #vm_deviceunplug($vmid, $conf, $opt);
4586             } elsif ($opt eq 'vcpus') {
4587                 die "skip\n" if !$hotplug_features->{cpu};
4588                 qemu_cpu_hotplug($vmid, $conf, undef);
4589             } elsif ($opt eq 'balloon') {
4590                 # enable balloon device is not hotpluggable
4591                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4592                 # here we reset the ballooning value to memory
4593                 my $balloon = $conf->{memory} || $defaults->{memory};
4594                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4595             } elsif ($fast_plug_option->{$opt}) {
4596                 # do nothing
4597             } elsif ($opt =~ m/^net(\d+)$/) {
4598                 die "skip\n" if !$hotplug_features->{network};
4599                 vm_deviceunplug($vmid, $conf, $opt);
4600             } elsif (is_valid_drivename($opt)) {
4601                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4602                 vm_deviceunplug($vmid, $conf, $opt);
4603                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4604             } elsif ($opt =~ m/^memory$/) {
4605                 die "skip\n" if !$hotplug_features->{memory};
4606                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4607             } elsif ($opt eq 'cpuunits') {
4608                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4609             } elsif ($opt eq 'cpulimit') {
4610                 $cgroup->change_cpu_quota(-1, 100000);
4611             } else {
4612                 die "skip\n";
4613             }
4614         };
4615         if (my $err = $@) {
4616             &$add_error($opt, $err) if $err ne "skip\n";
4617         } else {
4618             delete $conf->{$opt};
4619             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4620         }
4621     }
4622
4623     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4624     $apply_pending_cloudinit = sub {
4625         return if $apply_pending_cloudinit_done; # once is enough
4626         $apply_pending_cloudinit_done = 1; # once is enough
4627
4628         my ($key, $value) = @_;
4629
4630         my @cloudinit_opts = keys %$confdesc_cloudinit;
4631         foreach my $opt (keys %{$conf->{pending}}) {
4632             next if !grep { $_ eq $opt } @cloudinit_opts;
4633             $conf->{$opt} = delete $conf->{pending}->{$opt};
4634         }
4635
4636         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4637         foreach my $opt (sort keys %$pending_delete_hash) {
4638             next if !grep { $_ eq $opt } @cloudinit_opts;
4639             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4640             delete $conf->{$opt};
4641         }
4642
4643         my $new_conf = { %$conf };
4644         $new_conf->{$key} = $value;
4645         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4646     };
4647
4648     foreach my $opt (keys %{$conf->{pending}}) {
4649         next if $selection && !$selection->{$opt};
4650         my $value = $conf->{pending}->{$opt};
4651         eval {
4652             if ($opt eq 'hotplug') {
4653                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4654             } elsif ($opt eq 'tablet') {
4655                 die "skip\n" if !$hotplug_features->{usb};
4656                 if ($value == 1) {
4657                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4658                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4659                         if $arch eq 'aarch64';
4660                 } elsif ($value == 0) {
4661                     vm_deviceunplug($vmid, $conf, 'tablet');
4662                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4663                 }
4664             } elsif ($opt =~ m/^usb\d+$/) {
4665                 die "skip\n";
4666                 # since we cannot reliably hot unplug usb devices we disable it for now
4667                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4668                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4669                 #die "skip\n" if !$d;
4670                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4671             } elsif ($opt eq 'vcpus') {
4672                 die "skip\n" if !$hotplug_features->{cpu};
4673                 qemu_cpu_hotplug($vmid, $conf, $value);
4674             } elsif ($opt eq 'balloon') {
4675                 # enable/disable balloning device is not hotpluggable
4676                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4677                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4678                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4679
4680                 # allow manual ballooning if shares is set to zero
4681                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4682                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4683                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4684                 }
4685             } elsif ($opt =~ m/^net(\d+)$/) {
4686                 # some changes can be done without hotplug
4687                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4688                                     $vmid, $opt, $value, $arch, $machine_type);
4689             } elsif (is_valid_drivename($opt)) {
4690                 die "skip\n" if $opt eq 'efidisk0';
4691                 # some changes can be done without hotplug
4692                 my $drive = parse_drive($opt, $value);
4693                 if (drive_is_cloudinit($drive)) {
4694                     &$apply_pending_cloudinit($opt, $value);
4695                 }
4696                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4697                                      $vmid, $opt, $value, $arch, $machine_type);
4698             } elsif ($opt =~ m/^memory$/) { #dimms
4699                 die "skip\n" if !$hotplug_features->{memory};
4700                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4701             } elsif ($opt eq 'cpuunits') {
4702                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4703             } elsif ($opt eq 'cpulimit') {
4704                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4705                 $cgroup->change_cpu_quota($cpulimit, 100000);
4706             } else {
4707                 die "skip\n";  # skip non-hot-pluggable options
4708             }
4709         };
4710         if (my $err = $@) {
4711             &$add_error($opt, $err) if $err ne "skip\n";
4712         } else {
4713             $conf->{$opt} = $value;
4714             delete $conf->{pending}->{$opt};
4715         }
4716     }
4717
4718     PVE::QemuConfig->write_config($vmid, $conf);
4719 }
4720
4721 sub try_deallocate_drive {
4722     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4723
4724     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4725         my $volid = $drive->{file};
4726         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4727             my $sid = PVE::Storage::parse_volume_id($volid);
4728             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4729
4730             # check if the disk is really unused
4731             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4732                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4733             PVE::Storage::vdisk_free($storecfg, $volid);
4734             return 1;
4735         } else {
4736             # If vm is not owner of this disk remove from config
4737             return 1;
4738         }
4739     }
4740
4741     return;
4742 }
4743
4744 sub vmconfig_delete_or_detach_drive {
4745     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4746
4747     my $drive = parse_drive($opt, $conf->{$opt});
4748
4749     my $rpcenv = PVE::RPCEnvironment::get();
4750     my $authuser = $rpcenv->get_user();
4751
4752     if ($force) {
4753         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4754         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4755     } else {
4756         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4757     }
4758 }
4759
4760
4761
4762 sub vmconfig_apply_pending {
4763     my ($vmid, $conf, $storecfg, $errors) = @_;
4764
4765     my $add_apply_error = sub {
4766         my ($opt, $msg) = @_;
4767         my $err_msg = "unable to apply pending change $opt : $msg";
4768         $errors->{$opt} = $err_msg;
4769         warn $err_msg;
4770     };
4771
4772     # cold plug
4773
4774     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4775     foreach my $opt (sort keys %$pending_delete_hash) {
4776         my $force = $pending_delete_hash->{$opt}->{force};
4777         eval {
4778             if ($opt =~ m/^unused/) {
4779                 die "internal error";
4780             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4781                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4782             }
4783         };
4784         if (my $err = $@) {
4785             $add_apply_error->($opt, $err);
4786         } else {
4787             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4788             delete $conf->{$opt};
4789         }
4790     }
4791
4792     PVE::QemuConfig->cleanup_pending($conf);
4793
4794     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4795         next if $opt eq 'delete'; # just to be sure
4796         eval {
4797             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4798                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4799             }
4800         };
4801         if (my $err = $@) {
4802             $add_apply_error->($opt, $err);
4803         } else {
4804             $conf->{$opt} = delete $conf->{pending}->{$opt};
4805         }
4806     }
4807
4808     # write all changes at once to avoid unnecessary i/o
4809     PVE::QemuConfig->write_config($vmid, $conf);
4810 }
4811
4812 sub vmconfig_update_net {
4813     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4814
4815     my $newnet = parse_net($value);
4816
4817     if ($conf->{$opt}) {
4818         my $oldnet = parse_net($conf->{$opt});
4819
4820         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4821             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4822             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4823             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4824
4825             # for non online change, we try to hot-unplug
4826             die "skip\n" if !$hotplug;
4827             vm_deviceunplug($vmid, $conf, $opt);
4828         } else {
4829
4830             die "internal error" if $opt !~ m/net(\d+)/;
4831             my $iface = "tap${vmid}i$1";
4832
4833             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4834                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4835                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4836                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4837                 PVE::Network::tap_unplug($iface);
4838
4839                 if ($have_sdn) {
4840                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4841                 } else {
4842                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4843                 }
4844             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4845                 # Rate can be applied on its own but any change above needs to
4846                 # include the rate in tap_plug since OVS resets everything.
4847                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4848             }
4849
4850             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4851                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4852             }
4853
4854             return 1;
4855         }
4856     }
4857
4858     if ($hotplug) {
4859         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4860     } else {
4861         die "skip\n";
4862     }
4863 }
4864
4865 sub vmconfig_update_disk {
4866     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4867
4868     my $drive = parse_drive($opt, $value);
4869
4870     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4871         my $media = $drive->{media} || 'disk';
4872         my $oldmedia = $old_drive->{media} || 'disk';
4873         die "unable to change media type\n" if $media ne $oldmedia;
4874
4875         if (!drive_is_cdrom($old_drive)) {
4876
4877             if ($drive->{file} ne $old_drive->{file}) {
4878
4879                 die "skip\n" if !$hotplug;
4880
4881                 # unplug and register as unused
4882                 vm_deviceunplug($vmid, $conf, $opt);
4883                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4884
4885             } else {
4886                 # update existing disk
4887
4888                 # skip non hotpluggable value
4889                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4890                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4891                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4892                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4893                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4894                     die "skip\n";
4895                 }
4896
4897                 # apply throttle
4898                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4899                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4900                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4901                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4902                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4903                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4904                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4905                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4906                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4907                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4908                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4909                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4910                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4911                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4912                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4913                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4914                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4915                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4916
4917                     qemu_block_set_io_throttle(
4918                         $vmid,"drive-$opt",
4919                         ($drive->{mbps} || 0)*1024*1024,
4920                         ($drive->{mbps_rd} || 0)*1024*1024,
4921                         ($drive->{mbps_wr} || 0)*1024*1024,
4922                          $drive->{iops} || 0,
4923                          $drive->{iops_rd} || 0,
4924                          $drive->{iops_wr} || 0,
4925                         ($drive->{mbps_max} || 0)*1024*1024,
4926                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4927                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4928                          $drive->{iops_max} || 0,
4929                          $drive->{iops_rd_max} || 0,
4930                          $drive->{iops_wr_max} || 0,
4931                          $drive->{bps_max_length} || 1,
4932                          $drive->{bps_rd_max_length} || 1,
4933                          $drive->{bps_wr_max_length} || 1,
4934                          $drive->{iops_max_length} || 1,
4935                          $drive->{iops_rd_max_length} || 1,
4936                          $drive->{iops_wr_max_length} || 1,
4937                     );
4938
4939                 }
4940
4941                 return 1;
4942             }
4943
4944         } else { # cdrom
4945
4946             if ($drive->{file} eq 'none') {
4947                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4948                 if (drive_is_cloudinit($old_drive)) {
4949                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4950                 }
4951             } else {
4952                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4953
4954                 # force eject if locked
4955                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4956
4957                 if ($path) {
4958                     mon_cmd($vmid, "blockdev-change-medium",
4959                         id => "$opt", filename => "$path");
4960                 }
4961             }
4962
4963             return 1;
4964         }
4965     }
4966
4967     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4968     # hotplug new disks
4969     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4970     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4971 }
4972
4973 # called in locked context by incoming migration
4974 sub vm_migrate_get_nbd_disks {
4975     my ($storecfg, $conf, $replicated_volumes) = @_;
4976
4977     my $local_volumes = {};
4978     PVE::QemuConfig->foreach_volume($conf, sub {
4979         my ($ds, $drive) = @_;
4980
4981         return if drive_is_cdrom($drive);
4982
4983         my $volid = $drive->{file};
4984
4985         return if !$volid;
4986
4987         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4988
4989         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4990         return if $scfg->{shared};
4991
4992         # replicated disks re-use existing state via bitmap
4993         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4994         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4995     });
4996     return $local_volumes;
4997 }
4998
4999 # called in locked context by incoming migration
5000 sub vm_migrate_alloc_nbd_disks {
5001     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5002
5003     my $format = undef;
5004
5005     my $nbd = {};
5006     foreach my $opt (sort keys %$source_volumes) {
5007         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5008
5009         if ($use_existing) {
5010             $nbd->{$opt}->{drivestr} = print_drive($drive);
5011             $nbd->{$opt}->{volid} = $volid;
5012             $nbd->{$opt}->{replicated} = 1;
5013             next;
5014         }
5015
5016         # If a remote storage is specified and the format of the original
5017         # volume is not available there, fall back to the default format.
5018         # Otherwise use the same format as the original.
5019         if (!$storagemap->{identity}) {
5020             $storeid = map_storage($storagemap, $storeid);
5021             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5022             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5023             my $fileFormat = qemu_img_format($scfg, $volname);
5024             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5025         } else {
5026             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5027             $format = qemu_img_format($scfg, $volname);
5028         }
5029
5030         my $size = $drive->{size} / 1024;
5031         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5032         my $newdrive = $drive;
5033         $newdrive->{format} = $format;
5034         $newdrive->{file} = $newvolid;
5035         my $drivestr = print_drive($newdrive);
5036         $nbd->{$opt}->{drivestr} = $drivestr;
5037         $nbd->{$opt}->{volid} = $newvolid;
5038     }
5039
5040     return $nbd;
5041 }
5042
5043 # see vm_start_nolock for parameters, additionally:
5044 # migrate_opts:
5045 #   storagemap = parsed storage map for allocating NBD disks
5046 sub vm_start {
5047     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5048
5049     return PVE::QemuConfig->lock_config($vmid, sub {
5050         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5051
5052         die "you can't start a vm if it's a template\n"
5053             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5054
5055         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5056         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5057
5058         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5059
5060         if ($has_backup_lock && $running) {
5061             # a backup is currently running, attempt to start the guest in the
5062             # existing QEMU instance
5063             return vm_resume($vmid);
5064         }
5065
5066         PVE::QemuConfig->check_lock($conf)
5067             if !($params->{skiplock} || $has_suspended_lock);
5068
5069         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5070
5071         die "VM $vmid already running\n" if $running;
5072
5073         if (my $storagemap = $migrate_opts->{storagemap}) {
5074             my $replicated = $migrate_opts->{replicated_volumes};
5075             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5076             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5077
5078             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5079                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5080             }
5081         }
5082
5083         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5084     });
5085 }
5086
5087
5088 # params:
5089 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5090 #   skiplock => 0/1, skip checking for config lock
5091 #   skiptemplate => 0/1, skip checking whether VM is template
5092 #   forcemachine => to force Qemu machine (rollback/migration)
5093 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5094 #   timeout => in seconds
5095 #   paused => start VM in paused state (backup)
5096 #   resume => resume from hibernation
5097 #   pbs-backing => {
5098 #      sata0 => {
5099 #         repository
5100 #         snapshot
5101 #         keyfile
5102 #         archive
5103 #      },
5104 #      virtio2 => ...
5105 #   }
5106 # migrate_opts:
5107 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5108 #   migratedfrom => source node
5109 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5110 #   network => CIDR of migration network
5111 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5112 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5113 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5114 sub vm_start_nolock {
5115     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5116
5117     my $statefile = $params->{statefile};
5118     my $resume = $params->{resume};
5119
5120     my $migratedfrom = $migrate_opts->{migratedfrom};
5121     my $migration_type = $migrate_opts->{type};
5122
5123     my $res = {};
5124
5125     # clean up leftover reboot request files
5126     eval { clear_reboot_request($vmid); };
5127     warn $@ if $@;
5128
5129     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5130         vmconfig_apply_pending($vmid, $conf, $storecfg);
5131         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5132     }
5133
5134     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5135
5136     my $defaults = load_defaults();
5137
5138     # set environment variable useful inside network script
5139     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5140
5141     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5142
5143     my $forcemachine = $params->{forcemachine};
5144     my $forcecpu = $params->{forcecpu};
5145     if ($resume) {
5146         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5147         $forcemachine = $conf->{runningmachine};
5148         $forcecpu = $conf->{runningcpu};
5149         print "Resuming suspended VM\n";
5150     }
5151
5152     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5153         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5154
5155     my $migration_ip;
5156     my $get_migration_ip = sub {
5157         my ($nodename) = @_;
5158
5159         return $migration_ip if defined($migration_ip);
5160
5161         my $cidr = $migrate_opts->{network};
5162
5163         if (!defined($cidr)) {
5164             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5165             $cidr = $dc_conf->{migration}->{network};
5166         }
5167
5168         if (defined($cidr)) {
5169             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5170
5171             die "could not get IP: no address configured on local " .
5172                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5173
5174             die "could not get IP: multiple addresses configured on local " .
5175                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5176
5177             $migration_ip = @$ips[0];
5178         }
5179
5180         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5181             if !defined($migration_ip);
5182
5183         return $migration_ip;
5184     };
5185
5186     my $migrate_uri;
5187     if ($statefile) {
5188         if ($statefile eq 'tcp') {
5189             my $localip = "localhost";
5190             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5191             my $nodename = nodename();
5192
5193             if (!defined($migration_type)) {
5194                 if (defined($datacenterconf->{migration}->{type})) {
5195                     $migration_type = $datacenterconf->{migration}->{type};
5196                 } else {
5197                     $migration_type = 'secure';
5198                 }
5199             }
5200
5201             if ($migration_type eq 'insecure') {
5202                 $localip = $get_migration_ip->($nodename);
5203                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5204             }
5205
5206             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5207             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5208             $migrate_uri = "tcp:${localip}:${migrate_port}";
5209             push @$cmd, '-incoming', $migrate_uri;
5210             push @$cmd, '-S';
5211
5212         } elsif ($statefile eq 'unix') {
5213             # should be default for secure migrations as a ssh TCP forward
5214             # tunnel is not deterministic reliable ready and fails regurarly
5215             # to set up in time, so use UNIX socket forwards
5216             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5217             unlink $socket_addr;
5218
5219             $migrate_uri = "unix:$socket_addr";
5220
5221             push @$cmd, '-incoming', $migrate_uri;
5222             push @$cmd, '-S';
5223
5224         } elsif (-e $statefile) {
5225             push @$cmd, '-loadstate', $statefile;
5226         } else {
5227             my $statepath = PVE::Storage::path($storecfg, $statefile);
5228             push @$vollist, $statefile;
5229             push @$cmd, '-loadstate', $statepath;
5230         }
5231     } elsif ($params->{paused}) {
5232         push @$cmd, '-S';
5233     }
5234
5235     # host pci devices
5236     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5237       my $d = parse_hostpci($conf->{"hostpci$i"});
5238       next if !$d;
5239       my $pcidevices = $d->{pciid};
5240       foreach my $pcidevice (@$pcidevices) {
5241             my $pciid = $pcidevice->{id};
5242
5243             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5244             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5245             die "no pci device info for device '$pciid'\n" if !$info;
5246
5247             if ($d->{mdev}) {
5248                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5249                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5250             } else {
5251                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5252                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5253                 die "can't reset PCI device '$pciid'\n"
5254                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5255             }
5256       }
5257     }
5258
5259     PVE::Storage::activate_volumes($storecfg, $vollist);
5260
5261     eval {
5262         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5263             outfunc => sub {}, errfunc => sub {});
5264     };
5265     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5266     # timeout should be more than enough here...
5267     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5268
5269     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5270                                               : $defaults->{cpuunits};
5271
5272     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5273     my %run_params = (
5274         timeout => $statefile ? undef : $start_timeout,
5275         umask => 0077,
5276         noerr => 1,
5277     );
5278
5279     # when migrating, prefix QEMU output so other side can pick up any
5280     # errors that might occur and show the user
5281     if ($migratedfrom) {
5282         $run_params{quiet} = 1;
5283         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5284     }
5285
5286     my %properties = (
5287         Slice => 'qemu.slice',
5288         KillMode => 'process',
5289         SendSIGKILL => 0,
5290         TimeoutStopUSec => ULONG_MAX, # infinity
5291     );
5292
5293     if (PVE::CGroup::cgroup_mode() == 2) {
5294         $properties{CPUWeight} = $cpuunits;
5295     } else {
5296         $properties{CPUShares} = $cpuunits;
5297     }
5298
5299     if (my $cpulimit = $conf->{cpulimit}) {
5300         $properties{CPUQuota} = int($cpulimit * 100);
5301     }
5302     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5303
5304     my $run_qemu = sub {
5305         PVE::Tools::run_fork sub {
5306             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5307
5308             my $exitcode = run_command($cmd, %run_params);
5309             die "QEMU exited with code $exitcode\n" if $exitcode;
5310         };
5311     };
5312
5313     if ($conf->{hugepages}) {
5314
5315         my $code = sub {
5316             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5317             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5318
5319             PVE::QemuServer::Memory::hugepages_mount();
5320             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5321
5322             eval { $run_qemu->() };
5323             if (my $err = $@) {
5324                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5325                     if !$conf->{keephugepages};
5326                 die $err;
5327             }
5328
5329             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5330                 if !$conf->{keephugepages};
5331         };
5332         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5333
5334     } else {
5335         eval { $run_qemu->() };
5336     }
5337
5338     if (my $err = $@) {
5339         # deactivate volumes if start fails
5340         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5341         die "start failed: $err";
5342     }
5343
5344     print "migration listens on $migrate_uri\n" if $migrate_uri;
5345     $res->{migrate_uri} = $migrate_uri;
5346
5347     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5348         eval { mon_cmd($vmid, "cont"); };
5349         warn $@ if $@;
5350     }
5351
5352     #start nbd server for storage migration
5353     if (my $nbd = $migrate_opts->{nbd}) {
5354         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5355
5356         my $migrate_storage_uri;
5357         # nbd_protocol_version > 0 for unix socket support
5358         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5359             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5360             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5361             $migrate_storage_uri = "nbd:unix:$socket_path";
5362         } else {
5363             my $nodename = nodename();
5364             my $localip = $get_migration_ip->($nodename);
5365             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5366             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5367
5368             mon_cmd($vmid, "nbd-server-start", addr => {
5369                 type => 'inet',
5370                 data => {
5371                     host => "${localip}",
5372                     port => "${storage_migrate_port}",
5373                 },
5374             });
5375             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5376             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5377         }
5378
5379         $res->{migrate_storage_uri} = $migrate_storage_uri;
5380
5381         foreach my $opt (sort keys %$nbd) {
5382             my $drivestr = $nbd->{$opt}->{drivestr};
5383             my $volid = $nbd->{$opt}->{volid};
5384             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5385             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5386             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5387             print "re-using replicated volume: $opt - $volid\n"
5388                 if $nbd->{$opt}->{replicated};
5389
5390             $res->{drives}->{$opt} = $nbd->{$opt};
5391             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5392         }
5393     }
5394
5395     if ($migratedfrom) {
5396         eval {
5397             set_migration_caps($vmid);
5398         };
5399         warn $@ if $@;
5400
5401         if ($spice_port) {
5402             print "spice listens on port $spice_port\n";
5403             $res->{spice_port} = $spice_port;
5404             if ($migrate_opts->{spice_ticket}) {
5405                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5406                     $migrate_opts->{spice_ticket});
5407                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5408             }
5409         }
5410
5411     } else {
5412         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5413             if !$statefile && $conf->{balloon};
5414
5415         foreach my $opt (keys %$conf) {
5416             next if $opt !~  m/^net\d+$/;
5417             my $nicconf = parse_net($conf->{$opt});
5418             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5419         }
5420     }
5421
5422     mon_cmd($vmid, 'qom-set',
5423                 path => "machine/peripheral/balloon0",
5424                 property => "guest-stats-polling-interval",
5425                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5426
5427     if ($resume) {
5428         print "Resumed VM, removing state\n";
5429         if (my $vmstate = $conf->{vmstate}) {
5430             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5431             PVE::Storage::vdisk_free($storecfg, $vmstate);
5432         }
5433         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5434         PVE::QemuConfig->write_config($vmid, $conf);
5435     }
5436
5437     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5438
5439     return $res;
5440 }
5441
5442 sub vm_commandline {
5443     my ($storecfg, $vmid, $snapname) = @_;
5444
5445     my $conf = PVE::QemuConfig->load_config($vmid);
5446     my $forcemachine;
5447     my $forcecpu;
5448
5449     if ($snapname) {
5450         my $snapshot = $conf->{snapshots}->{$snapname};
5451         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5452
5453         # check for machine or CPU overrides in snapshot
5454         $forcemachine = $snapshot->{runningmachine};
5455         $forcecpu = $snapshot->{runningcpu};
5456
5457         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5458
5459         $conf = $snapshot;
5460     }
5461
5462     my $defaults = load_defaults();
5463
5464     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5465         $forcemachine, $forcecpu);
5466
5467     return PVE::Tools::cmd2string($cmd);
5468 }
5469
5470 sub vm_reset {
5471     my ($vmid, $skiplock) = @_;
5472
5473     PVE::QemuConfig->lock_config($vmid, sub {
5474
5475         my $conf = PVE::QemuConfig->load_config($vmid);
5476
5477         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5478
5479         mon_cmd($vmid, "system_reset");
5480     });
5481 }
5482
5483 sub get_vm_volumes {
5484     my ($conf) = @_;
5485
5486     my $vollist = [];
5487     foreach_volid($conf, sub {
5488         my ($volid, $attr) = @_;
5489
5490         return if $volid =~ m|^/|;
5491
5492         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5493         return if !$sid;
5494
5495         push @$vollist, $volid;
5496     });
5497
5498     return $vollist;
5499 }
5500
5501 sub vm_stop_cleanup {
5502     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5503
5504     eval {
5505
5506         if (!$keepActive) {
5507             my $vollist = get_vm_volumes($conf);
5508             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5509         }
5510
5511         foreach my $ext (qw(mon qmp pid vnc qga)) {
5512             unlink "/var/run/qemu-server/${vmid}.$ext";
5513         }
5514
5515         if ($conf->{ivshmem}) {
5516             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5517             # just delete it for now, VMs which have this already open do not
5518             # are affected, but new VMs will get a separated one. If this
5519             # becomes an issue we either add some sort of ref-counting or just
5520             # add a "don't delete on stop" flag to the ivshmem format.
5521             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5522         }
5523
5524         foreach my $key (keys %$conf) {
5525             next if $key !~ m/^hostpci(\d+)$/;
5526             my $hostpciindex = $1;
5527             my $d = parse_hostpci($conf->{$key});
5528             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5529
5530             foreach my $pci (@{$d->{pciid}}) {
5531                 my $pciid = $pci->{id};
5532                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5533             }
5534         }
5535
5536         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5537     };
5538     warn $@ if $@; # avoid errors - just warn
5539 }
5540
5541 # call only in locked context
5542 sub _do_vm_stop {
5543     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5544
5545     my $pid = check_running($vmid, $nocheck);
5546     return if !$pid;
5547
5548     my $conf;
5549     if (!$nocheck) {
5550         $conf = PVE::QemuConfig->load_config($vmid);
5551         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5552         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5553             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5554             $timeout = $opts->{down} if $opts->{down};
5555         }
5556         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5557     }
5558
5559     eval {
5560         if ($shutdown) {
5561             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5562                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5563             } else {
5564                 mon_cmd($vmid, "system_powerdown");
5565             }
5566         } else {
5567             mon_cmd($vmid, "quit");
5568         }
5569     };
5570     my $err = $@;
5571
5572     if (!$err) {
5573         $timeout = 60 if !defined($timeout);
5574
5575         my $count = 0;
5576         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5577             $count++;
5578             sleep 1;
5579         }
5580
5581         if ($count >= $timeout) {
5582             if ($force) {
5583                 warn "VM still running - terminating now with SIGTERM\n";
5584                 kill 15, $pid;
5585             } else {
5586                 die "VM quit/powerdown failed - got timeout\n";
5587             }
5588         } else {
5589             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5590             return;
5591         }
5592     } else {
5593        if (!check_running($vmid, $nocheck)) {
5594             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5595             return;
5596        }
5597        if ($force) {
5598             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5599             kill 15, $pid;
5600         } else {
5601             die "VM quit/powerdown failed\n";
5602         }
5603     }
5604
5605     # wait again
5606     $timeout = 10;
5607
5608     my $count = 0;
5609     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5610         $count++;
5611         sleep 1;
5612     }
5613
5614     if ($count >= $timeout) {
5615         warn "VM still running - terminating now with SIGKILL\n";
5616         kill 9, $pid;
5617         sleep 1;
5618     }
5619
5620     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5621 }
5622
5623 # Note: use $nocheck to skip tests if VM configuration file exists.
5624 # We need that when migration VMs to other nodes (files already moved)
5625 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5626 sub vm_stop {
5627     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5628
5629     $force = 1 if !defined($force) && !$shutdown;
5630
5631     if ($migratedfrom){
5632         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5633         kill 15, $pid if $pid;
5634         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5635         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5636         return;
5637     }
5638
5639     PVE::QemuConfig->lock_config($vmid, sub {
5640         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5641    });
5642 }
5643
5644 sub vm_reboot {
5645     my ($vmid, $timeout) = @_;
5646
5647     PVE::QemuConfig->lock_config($vmid, sub {
5648         eval {
5649
5650             # only reboot if running, as qmeventd starts it again on a stop event
5651             return if !check_running($vmid);
5652
5653             create_reboot_request($vmid);
5654
5655             my $storecfg = PVE::Storage::config();
5656             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5657
5658         };
5659         if (my $err = $@) {
5660             # avoid that the next normal shutdown will be confused for a reboot
5661             clear_reboot_request($vmid);
5662             die $err;
5663         }
5664    });
5665 }
5666
5667 # note: if using the statestorage parameter, the caller has to check privileges
5668 sub vm_suspend {
5669     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5670
5671     my $conf;
5672     my $path;
5673     my $storecfg;
5674     my $vmstate;
5675
5676     PVE::QemuConfig->lock_config($vmid, sub {
5677
5678         $conf = PVE::QemuConfig->load_config($vmid);
5679
5680         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5681         PVE::QemuConfig->check_lock($conf)
5682             if !($skiplock || $is_backing_up);
5683
5684         die "cannot suspend to disk during backup\n"
5685             if $is_backing_up && $includestate;
5686
5687         if ($includestate) {
5688             $conf->{lock} = 'suspending';
5689             my $date = strftime("%Y-%m-%d", localtime(time()));
5690             $storecfg = PVE::Storage::config();
5691             if (!$statestorage) {
5692                 $statestorage = find_vmstate_storage($conf, $storecfg);
5693                 # check permissions for the storage
5694                 my $rpcenv = PVE::RPCEnvironment::get();
5695                 if ($rpcenv->{type} ne 'cli') {
5696                     my $authuser = $rpcenv->get_user();
5697                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5698                 }
5699             }
5700
5701
5702             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5703                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5704             $path = PVE::Storage::path($storecfg, $vmstate);
5705             PVE::QemuConfig->write_config($vmid, $conf);
5706         } else {
5707             mon_cmd($vmid, "stop");
5708         }
5709     });
5710
5711     if ($includestate) {
5712         # save vm state
5713         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5714
5715         eval {
5716             set_migration_caps($vmid, 1);
5717             mon_cmd($vmid, "savevm-start", statefile => $path);
5718             for(;;) {
5719                 my $state = mon_cmd($vmid, "query-savevm");
5720                 if (!$state->{status}) {
5721                     die "savevm not active\n";
5722                 } elsif ($state->{status} eq 'active') {
5723                     sleep(1);
5724                     next;
5725                 } elsif ($state->{status} eq 'completed') {
5726                     print "State saved, quitting\n";
5727                     last;
5728                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5729                     die "query-savevm failed with error '$state->{error}'\n"
5730                 } else {
5731                     die "query-savevm returned status '$state->{status}'\n";
5732                 }
5733             }
5734         };
5735         my $err = $@;
5736
5737         PVE::QemuConfig->lock_config($vmid, sub {
5738             $conf = PVE::QemuConfig->load_config($vmid);
5739             if ($err) {
5740                 # cleanup, but leave suspending lock, to indicate something went wrong
5741                 eval {
5742                     mon_cmd($vmid, "savevm-end");
5743                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5744                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5745                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5746                     PVE::QemuConfig->write_config($vmid, $conf);
5747                 };
5748                 warn $@ if $@;
5749                 die $err;
5750             }
5751
5752             die "lock changed unexpectedly\n"
5753                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5754
5755             mon_cmd($vmid, "quit");
5756             $conf->{lock} = 'suspended';
5757             PVE::QemuConfig->write_config($vmid, $conf);
5758         });
5759     }
5760 }
5761
5762 sub vm_resume {
5763     my ($vmid, $skiplock, $nocheck) = @_;
5764
5765     PVE::QemuConfig->lock_config($vmid, sub {
5766         my $res = mon_cmd($vmid, 'query-status');
5767         my $resume_cmd = 'cont';
5768         my $reset = 0;
5769
5770         if ($res->{status}) {
5771             return if $res->{status} eq 'running'; # job done, go home
5772             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5773             $reset = 1 if $res->{status} eq 'shutdown';
5774         }
5775
5776         if (!$nocheck) {
5777
5778             my $conf = PVE::QemuConfig->load_config($vmid);
5779
5780             PVE::QemuConfig->check_lock($conf)
5781                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5782         }
5783
5784         if ($reset) {
5785             # required if a VM shuts down during a backup and we get a resume
5786             # request before the backup finishes for example
5787             mon_cmd($vmid, "system_reset");
5788         }
5789         mon_cmd($vmid, $resume_cmd);
5790     });
5791 }
5792
5793 sub vm_sendkey {
5794     my ($vmid, $skiplock, $key) = @_;
5795
5796     PVE::QemuConfig->lock_config($vmid, sub {
5797
5798         my $conf = PVE::QemuConfig->load_config($vmid);
5799
5800         # there is no qmp command, so we use the human monitor command
5801         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5802         die $res if $res ne '';
5803     });
5804 }
5805
5806 # vzdump restore implementaion
5807
5808 sub tar_archive_read_firstfile {
5809     my $archive = shift;
5810
5811     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5812
5813     # try to detect archive type first
5814     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5815         die "unable to open file '$archive'\n";
5816     my $firstfile = <$fh>;
5817     kill 15, $pid;
5818     close $fh;
5819
5820     die "ERROR: archive contaions no data\n" if !$firstfile;
5821     chomp $firstfile;
5822
5823     return $firstfile;
5824 }
5825
5826 sub tar_restore_cleanup {
5827     my ($storecfg, $statfile) = @_;
5828
5829     print STDERR "starting cleanup\n";
5830
5831     if (my $fd = IO::File->new($statfile, "r")) {
5832         while (defined(my $line = <$fd>)) {
5833             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5834                 my $volid = $2;
5835                 eval {
5836                     if ($volid =~ m|^/|) {
5837                         unlink $volid || die 'unlink failed\n';
5838                     } else {
5839                         PVE::Storage::vdisk_free($storecfg, $volid);
5840                     }
5841                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5842                 };
5843                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5844             } else {
5845                 print STDERR "unable to parse line in statfile - $line";
5846             }
5847         }
5848         $fd->close();
5849     }
5850 }
5851
5852 sub restore_file_archive {
5853     my ($archive, $vmid, $user, $opts) = @_;
5854
5855     return restore_vma_archive($archive, $vmid, $user, $opts)
5856         if $archive eq '-';
5857
5858     my $info = PVE::Storage::archive_info($archive);
5859     my $format = $opts->{format} // $info->{format};
5860     my $comp = $info->{compression};
5861
5862     # try to detect archive format
5863     if ($format eq 'tar') {
5864         return restore_tar_archive($archive, $vmid, $user, $opts);
5865     } else {
5866         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5867     }
5868 }
5869
5870 # hepler to remove disks that will not be used after restore
5871 my $restore_cleanup_oldconf = sub {
5872     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5873
5874     PVE::QemuConfig->foreach_volume($oldconf, sub {
5875         my ($ds, $drive) = @_;
5876
5877         return if drive_is_cdrom($drive, 1);
5878
5879         my $volid = $drive->{file};
5880         return if !$volid || $volid =~ m|^/|;
5881
5882         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5883         return if !$path || !$owner || ($owner != $vmid);
5884
5885         # Note: only delete disk we want to restore
5886         # other volumes will become unused
5887         if ($virtdev_hash->{$ds}) {
5888             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5889             if (my $err = $@) {
5890                 warn $err;
5891             }
5892         }
5893     });
5894
5895     # delete vmstate files, after the restore we have no snapshots anymore
5896     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5897         my $snap = $oldconf->{snapshots}->{$snapname};
5898         if ($snap->{vmstate}) {
5899             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5900             if (my $err = $@) {
5901                 warn $err;
5902             }
5903         }
5904     }
5905 };
5906
5907 # Helper to parse vzdump backup device hints
5908 #
5909 # $rpcenv: Environment, used to ckeck storage permissions
5910 # $user: User ID, to check storage permissions
5911 # $storecfg: Storage configuration
5912 # $fh: the file handle for reading the configuration
5913 # $devinfo: should contain device sizes for all backu-up'ed devices
5914 # $options: backup options (pool, default storage)
5915 #
5916 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5917 my $parse_backup_hints = sub {
5918     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5919
5920     my $virtdev_hash = {};
5921
5922     while (defined(my $line = <$fh>)) {
5923         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5924             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5925             die "archive does not contain data for drive '$virtdev'\n"
5926                 if !$devinfo->{$devname};
5927
5928             if (defined($options->{storage})) {
5929                 $storeid = $options->{storage} || 'local';
5930             } elsif (!$storeid) {
5931                 $storeid = 'local';
5932             }
5933             $format = 'raw' if !$format;
5934             $devinfo->{$devname}->{devname} = $devname;
5935             $devinfo->{$devname}->{virtdev} = $virtdev;
5936             $devinfo->{$devname}->{format} = $format;
5937             $devinfo->{$devname}->{storeid} = $storeid;
5938
5939             # check permission on storage
5940             my $pool = $options->{pool}; # todo: do we need that?
5941             if ($user ne 'root@pam') {
5942                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5943             }
5944
5945             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5946         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5947             my $virtdev = $1;
5948             my $drive = parse_drive($virtdev, $2);
5949             if (drive_is_cloudinit($drive)) {
5950                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5951                 $storeid = $options->{storage} if defined ($options->{storage});
5952                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5953                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5954
5955                 $virtdev_hash->{$virtdev} = {
5956                     format => $format,
5957                     storeid => $storeid,
5958                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5959                     is_cloudinit => 1,
5960                 };
5961             }
5962         }
5963     }
5964
5965     return $virtdev_hash;
5966 };
5967
5968 # Helper to allocate and activate all volumes required for a restore
5969 #
5970 # $storecfg: Storage configuration
5971 # $virtdev_hash: as returned by parse_backup_hints()
5972 #
5973 # Returns: { $virtdev => $volid }
5974 my $restore_allocate_devices = sub {
5975     my ($storecfg, $virtdev_hash, $vmid) = @_;
5976
5977     my $map = {};
5978     foreach my $virtdev (sort keys %$virtdev_hash) {
5979         my $d = $virtdev_hash->{$virtdev};
5980         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5981         my $storeid = $d->{storeid};
5982         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5983
5984         # test if requested format is supported
5985         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5986         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5987         $d->{format} = $defFormat if !$supported;
5988
5989         my $name;
5990         if ($d->{is_cloudinit}) {
5991             $name = "vm-$vmid-cloudinit";
5992             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5993             if ($scfg->{path}) {
5994                 $name .= ".$d->{format}";
5995             }
5996         }
5997
5998         my $volid = PVE::Storage::vdisk_alloc(
5999             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6000
6001         print STDERR "new volume ID is '$volid'\n";
6002         $d->{volid} = $volid;
6003
6004         PVE::Storage::activate_volumes($storecfg, [$volid]);
6005
6006         $map->{$virtdev} = $volid;
6007     }
6008
6009     return $map;
6010 };
6011
6012 sub restore_update_config_line {
6013     my ($cookie, $map, $line, $unique) = @_;
6014
6015     return '' if $line =~ m/^\#qmdump\#/;
6016     return '' if $line =~ m/^\#vzdump\#/;
6017     return '' if $line =~ m/^lock:/;
6018     return '' if $line =~ m/^unused\d+:/;
6019     return '' if $line =~ m/^parent:/;
6020
6021     my $res = '';
6022
6023     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6024     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6025         # try to convert old 1.X settings
6026         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6027         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6028             my ($model, $macaddr) = split(/\=/, $devconfig);
6029             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6030             my $net = {
6031                 model => $model,
6032                 bridge => "vmbr$ind",
6033                 macaddr => $macaddr,
6034             };
6035             my $netstr = print_net($net);
6036
6037             $res .= "net$cookie->{netcount}: $netstr\n";
6038             $cookie->{netcount}++;
6039         }
6040     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6041         my ($id, $netstr) = ($1, $2);
6042         my $net = parse_net($netstr);
6043         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6044         $netstr = print_net($net);
6045         $res .= "$id: $netstr\n";
6046     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6047         my $virtdev = $1;
6048         my $value = $3;
6049         my $di = parse_drive($virtdev, $value);
6050         if (defined($di->{backup}) && !$di->{backup}) {
6051             $res .= "#$line";
6052         } elsif ($map->{$virtdev}) {
6053             delete $di->{format}; # format can change on restore
6054             $di->{file} = $map->{$virtdev};
6055             $value = print_drive($di);
6056             $res .= "$virtdev: $value\n";
6057         } else {
6058             $res .= $line;
6059         }
6060     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6061         my $vmgenid = $1;
6062         if ($vmgenid ne '0') {
6063             # always generate a new vmgenid if there was a valid one setup
6064             $vmgenid = generate_uuid();
6065         }
6066         $res .= "vmgenid: $vmgenid\n";
6067     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6068         my ($uuid, $uuid_str);
6069         UUID::generate($uuid);
6070         UUID::unparse($uuid, $uuid_str);
6071         my $smbios1 = parse_smbios1($2);
6072         $smbios1->{uuid} = $uuid_str;
6073         $res .= $1.print_smbios1($smbios1)."\n";
6074     } else {
6075         $res .= $line;
6076     }
6077
6078     return $res;
6079 }
6080
6081 my $restore_deactivate_volumes = sub {
6082     my ($storecfg, $devinfo) = @_;
6083
6084     my $vollist = [];
6085     foreach my $devname (keys %$devinfo) {
6086         my $volid = $devinfo->{$devname}->{volid};
6087         push @$vollist, $volid if $volid;
6088     }
6089
6090     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6091 };
6092
6093 my $restore_destroy_volumes = sub {
6094     my ($storecfg, $devinfo) = @_;
6095
6096     foreach my $devname (keys %$devinfo) {
6097         my $volid = $devinfo->{$devname}->{volid};
6098         next if !$volid;
6099         eval {
6100             if ($volid =~ m|^/|) {
6101                 unlink $volid || die 'unlink failed\n';
6102             } else {
6103                 PVE::Storage::vdisk_free($storecfg, $volid);
6104             }
6105             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6106         };
6107         print STDERR "unable to cleanup '$volid' - $@" if $@;
6108     }
6109 };
6110
6111 sub scan_volids {
6112     my ($cfg, $vmid) = @_;
6113
6114     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6115
6116     my $volid_hash = {};
6117     foreach my $storeid (keys %$info) {
6118         foreach my $item (@{$info->{$storeid}}) {
6119             next if !($item->{volid} && $item->{size});
6120             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6121             $volid_hash->{$item->{volid}} = $item;
6122         }
6123     }
6124
6125     return $volid_hash;
6126 }
6127
6128 sub update_disk_config {
6129     my ($vmid, $conf, $volid_hash) = @_;
6130
6131     my $changes;
6132     my $prefix = "VM $vmid";
6133
6134     # used and unused disks
6135     my $referenced = {};
6136
6137     # Note: it is allowed to define multiple storages with same path (alias), so
6138     # we need to check both 'volid' and real 'path' (two different volid can point
6139     # to the same path).
6140
6141     my $referencedpath = {};
6142
6143     # update size info
6144     PVE::QemuConfig->foreach_volume($conf, sub {
6145         my ($opt, $drive) = @_;
6146
6147         my $volid = $drive->{file};
6148         return if !$volid;
6149         my $volume = $volid_hash->{$volid};
6150
6151         # mark volid as "in-use" for next step
6152         $referenced->{$volid} = 1;
6153         if ($volume && (my $path = $volume->{path})) {
6154             $referencedpath->{$path} = 1;
6155         }
6156
6157         return if drive_is_cdrom($drive);
6158         return if !$volume;
6159
6160         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6161         if (defined($updated)) {
6162             $changes = 1;
6163             $conf->{$opt} = print_drive($updated);
6164             print "$prefix ($opt): $msg\n";
6165         }
6166     });
6167
6168     # remove 'unusedX' entry if volume is used
6169     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6170         my ($opt, $drive) = @_;
6171
6172         my $volid = $drive->{file};
6173         return if !$volid;
6174
6175         my $path;
6176         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6177         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6178             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6179             $changes = 1;
6180             delete $conf->{$opt};
6181         }
6182
6183         $referenced->{$volid} = 1;
6184         $referencedpath->{$path} = 1 if $path;
6185     });
6186
6187     foreach my $volid (sort keys %$volid_hash) {
6188         next if $volid =~ m/vm-$vmid-state-/;
6189         next if $referenced->{$volid};
6190         my $path = $volid_hash->{$volid}->{path};
6191         next if !$path; # just to be sure
6192         next if $referencedpath->{$path};
6193         $changes = 1;
6194         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6195         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6196         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6197     }
6198
6199     return $changes;
6200 }
6201
6202 sub rescan {
6203     my ($vmid, $nolock, $dryrun) = @_;
6204
6205     my $cfg = PVE::Storage::config();
6206
6207     print "rescan volumes...\n";
6208     my $volid_hash = scan_volids($cfg, $vmid);
6209
6210     my $updatefn =  sub {
6211         my ($vmid) = @_;
6212
6213         my $conf = PVE::QemuConfig->load_config($vmid);
6214
6215         PVE::QemuConfig->check_lock($conf);
6216
6217         my $vm_volids = {};
6218         foreach my $volid (keys %$volid_hash) {
6219             my $info = $volid_hash->{$volid};
6220             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6221         }
6222
6223         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6224
6225         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6226     };
6227
6228     if (defined($vmid)) {
6229         if ($nolock) {
6230             &$updatefn($vmid);
6231         } else {
6232             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6233         }
6234     } else {
6235         my $vmlist = config_list();
6236         foreach my $vmid (keys %$vmlist) {
6237             if ($nolock) {
6238                 &$updatefn($vmid);
6239             } else {
6240                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6241             }
6242         }
6243     }
6244 }
6245
6246 sub restore_proxmox_backup_archive {
6247     my ($archive, $vmid, $user, $options) = @_;
6248
6249     my $storecfg = PVE::Storage::config();
6250
6251     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6252     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6253
6254     my $fingerprint = $scfg->{fingerprint};
6255     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6256
6257     my $repo = PVE::PBSClient::get_repository($scfg);
6258
6259     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6260     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6261     local $ENV{PBS_PASSWORD} = $password;
6262     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6263
6264     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6265         PVE::Storage::parse_volname($storecfg, $archive);
6266
6267     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6268
6269     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6270
6271     my $tmpdir = "/var/tmp/vzdumptmp$$";
6272     rmtree $tmpdir;
6273     mkpath $tmpdir;
6274
6275     my $conffile = PVE::QemuConfig->config_file($vmid);
6276      # disable interrupts (always do cleanups)
6277     local $SIG{INT} =
6278         local $SIG{TERM} =
6279         local $SIG{QUIT} =
6280         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6281
6282     # Note: $oldconf is undef if VM does not exists
6283     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6284     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6285     my $new_conf_raw = '';
6286
6287     my $rpcenv = PVE::RPCEnvironment::get();
6288     my $devinfo = {};
6289
6290     eval {
6291         # enable interrupts
6292         local $SIG{INT} =
6293             local $SIG{TERM} =
6294             local $SIG{QUIT} =
6295             local $SIG{HUP} =
6296             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6297
6298         my $cfgfn = "$tmpdir/qemu-server.conf";
6299         my $firewall_config_fn = "$tmpdir/fw.conf";
6300         my $index_fn = "$tmpdir/index.json";
6301
6302         my $cmd = "restore";
6303
6304         my $param = [$pbs_backup_name, "index.json", $index_fn];
6305         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6306         my $index = PVE::Tools::file_get_contents($index_fn);
6307         $index = decode_json($index);
6308
6309         # print Dumper($index);
6310         foreach my $info (@{$index->{files}}) {
6311             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6312                 my $devname = $1;
6313                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6314                     $devinfo->{$devname}->{size} = $1;
6315                 } else {
6316                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6317                 }
6318             }
6319         }
6320
6321         my $is_qemu_server_backup = scalar(
6322             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6323         );
6324         if (!$is_qemu_server_backup) {
6325             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6326         }
6327         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6328
6329         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6330         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6331
6332         if ($has_firewall_config) {
6333             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6334             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6335
6336             my $pve_firewall_dir = '/etc/pve/firewall';
6337             mkdir $pve_firewall_dir; # make sure the dir exists
6338             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6339         }
6340
6341         my $fh = IO::File->new($cfgfn, "r") ||
6342             die "unable to read qemu-server.conf - $!\n";
6343
6344         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6345
6346         # fixme: rate limit?
6347
6348         # create empty/temp config
6349         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6350
6351         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6352
6353         # allocate volumes
6354         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6355
6356         if (!$options->{live}) {
6357             foreach my $virtdev (sort keys %$virtdev_hash) {
6358                 my $d = $virtdev_hash->{$virtdev};
6359                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6360
6361                 my $volid = $d->{volid};
6362
6363                 my $path = PVE::Storage::path($storecfg, $volid);
6364
6365                 my $pbs_restore_cmd = [
6366                     '/usr/bin/pbs-restore',
6367                     '--repository', $repo,
6368                     $pbs_backup_name,
6369                     "$d->{devname}.img.fidx",
6370                     $path,
6371                     '--verbose',
6372                     ];
6373
6374                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6375                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6376
6377                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6378                     push @$pbs_restore_cmd, '--skip-zero';
6379                 }
6380
6381                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6382                 print "restore proxmox backup image: $dbg_cmdstring\n";
6383                 run_command($pbs_restore_cmd);
6384             }
6385         }
6386
6387         $fh->seek(0, 0) || die "seek failed - $!\n";
6388
6389         my $cookie = { netcount => 0 };
6390         while (defined(my $line = <$fh>)) {
6391             $new_conf_raw .= restore_update_config_line(
6392                 $cookie,
6393                 $map,
6394                 $line,
6395                 $options->{unique},
6396             );
6397         }
6398
6399         $fh->close();
6400     };
6401     my $err = $@;
6402
6403     if ($err || !$options->{live}) {
6404         $restore_deactivate_volumes->($storecfg, $devinfo);
6405     }
6406
6407     rmtree $tmpdir;
6408
6409     if ($err) {
6410         $restore_destroy_volumes->($storecfg, $devinfo);
6411         die $err;
6412     }
6413
6414     if ($options->{live}) {
6415         # keep lock during live-restore
6416         $new_conf_raw .= "\nlock: create";
6417     }
6418
6419     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6420
6421     PVE::Cluster::cfs_update(); # make sure we read new file
6422
6423     eval { rescan($vmid, 1); };
6424     warn $@ if $@;
6425
6426     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6427
6428     if ($options->{live}) {
6429         # enable interrupts
6430         local $SIG{INT} =
6431             local $SIG{TERM} =
6432             local $SIG{QUIT} =
6433             local $SIG{HUP} =
6434             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6435
6436         my $conf = PVE::QemuConfig->load_config($vmid);
6437         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6438
6439         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6440
6441         PVE::QemuConfig->remove_lock($vmid, "create");
6442     }
6443 }
6444
6445 sub pbs_live_restore {
6446     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6447
6448     print "starting VM for live-restore\n";
6449     print "repository: '$repo', snapshot: '$snap'\n";
6450
6451     my $pbs_backing = {};
6452     for my $ds (keys %$restored_disks) {
6453         $ds =~ m/^drive-(.*)$/;
6454         my $confname = $1;
6455         $pbs_backing->{$confname} = {
6456             repository => $repo,
6457             snapshot => $snap,
6458             archive => "$ds.img.fidx",
6459         };
6460         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6461
6462         my $drive = parse_drive($confname, $conf->{$confname});
6463         print "restoring '$ds' to '$drive->{file}'\n";
6464     }
6465
6466     my $drives_streamed = 0;
6467     eval {
6468         # make sure HA doesn't interrupt our restore by stopping the VM
6469         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6470             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6471         }
6472
6473         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6474         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6475         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6476
6477         my $qmeventd_fd = register_qmeventd_handle($vmid);
6478
6479         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6480         # this will effectively collapse the backing image chain consisting of
6481         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6482         # removes itself once all backing images vanish with 'auto-remove=on')
6483         my $jobs = {};
6484         for my $ds (sort keys %$restored_disks) {
6485             my $job_id = "restore-$ds";
6486             mon_cmd($vmid, 'block-stream',
6487                 'job-id' => $job_id,
6488                 device => "$ds",
6489             );
6490             $jobs->{$job_id} = {};
6491         }
6492
6493         mon_cmd($vmid, 'cont');
6494         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6495
6496         print "restore-drive jobs finished successfully, removing all tracking block devices"
6497             ." to disconnect from Proxmox Backup Server\n";
6498
6499         for my $ds (sort keys %$restored_disks) {
6500             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6501         }
6502
6503         close($qmeventd_fd);
6504     };
6505
6506     my $err = $@;
6507
6508     if ($err) {
6509         warn "An error occured during live-restore: $err\n";
6510         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6511         die "live-restore failed\n";
6512     }
6513 }
6514
6515 sub restore_vma_archive {
6516     my ($archive, $vmid, $user, $opts, $comp) = @_;
6517
6518     my $readfrom = $archive;
6519
6520     my $cfg = PVE::Storage::config();
6521     my $commands = [];
6522     my $bwlimit = $opts->{bwlimit};
6523
6524     my $dbg_cmdstring = '';
6525     my $add_pipe = sub {
6526         my ($cmd) = @_;
6527         push @$commands, $cmd;
6528         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6529         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6530         $readfrom = '-';
6531     };
6532
6533     my $input = undef;
6534     if ($archive eq '-') {
6535         $input = '<&STDIN';
6536     } else {
6537         # If we use a backup from a PVE defined storage we also consider that
6538         # storage's rate limit:
6539         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6540         if (defined($volid)) {
6541             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6542             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6543             if ($readlimit) {
6544                 print STDERR "applying read rate limit: $readlimit\n";
6545                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6546                 $add_pipe->($cstream);
6547             }
6548         }
6549     }
6550
6551     if ($comp) {
6552         my $info = PVE::Storage::decompressor_info('vma', $comp);
6553         my $cmd = $info->{decompressor};
6554         push @$cmd, $readfrom;
6555         $add_pipe->($cmd);
6556     }
6557
6558     my $tmpdir = "/var/tmp/vzdumptmp$$";
6559     rmtree $tmpdir;
6560
6561     # disable interrupts (always do cleanups)
6562     local $SIG{INT} =
6563         local $SIG{TERM} =
6564         local $SIG{QUIT} =
6565         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6566
6567     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6568     POSIX::mkfifo($mapfifo, 0600);
6569     my $fifofh;
6570     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6571
6572     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6573
6574     my $oldtimeout;
6575     my $timeout = 5;
6576
6577     my $devinfo = {};
6578
6579     my $rpcenv = PVE::RPCEnvironment::get();
6580
6581     my $conffile = PVE::QemuConfig->config_file($vmid);
6582
6583     # Note: $oldconf is undef if VM does not exist
6584     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6585     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6586     my $new_conf_raw = '';
6587
6588     my %storage_limits;
6589
6590     my $print_devmap = sub {
6591         my $cfgfn = "$tmpdir/qemu-server.conf";
6592
6593         # we can read the config - that is already extracted
6594         my $fh = IO::File->new($cfgfn, "r") ||
6595             die "unable to read qemu-server.conf - $!\n";
6596
6597         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6598         if (-f $fwcfgfn) {
6599             my $pve_firewall_dir = '/etc/pve/firewall';
6600             mkdir $pve_firewall_dir; # make sure the dir exists
6601             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6602         }
6603
6604         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6605
6606         foreach my $info (values %{$virtdev_hash}) {
6607             my $storeid = $info->{storeid};
6608             next if defined($storage_limits{$storeid});
6609
6610             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6611             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6612             $storage_limits{$storeid} = $limit * 1024;
6613         }
6614
6615         foreach my $devname (keys %$devinfo) {
6616             die "found no device mapping information for device '$devname'\n"
6617                 if !$devinfo->{$devname}->{virtdev};
6618         }
6619
6620         # create empty/temp config
6621         if ($oldconf) {
6622             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6623             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6624         }
6625
6626         # allocate volumes
6627         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6628
6629         # print restore information to $fifofh
6630         foreach my $virtdev (sort keys %$virtdev_hash) {
6631             my $d = $virtdev_hash->{$virtdev};
6632             next if $d->{is_cloudinit}; # no need to restore cloudinit
6633
6634             my $storeid = $d->{storeid};
6635             my $volid = $d->{volid};
6636
6637             my $map_opts = '';
6638             if (my $limit = $storage_limits{$storeid}) {
6639                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6640             }
6641
6642             my $write_zeros = 1;
6643             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6644                 $write_zeros = 0;
6645             }
6646
6647             my $path = PVE::Storage::path($cfg, $volid);
6648
6649             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6650
6651             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6652         }
6653
6654         $fh->seek(0, 0) || die "seek failed - $!\n";
6655
6656         my $cookie = { netcount => 0 };
6657         while (defined(my $line = <$fh>)) {
6658             $new_conf_raw .= restore_update_config_line(
6659                 $cookie,
6660                 $map,
6661                 $line,
6662                 $opts->{unique},
6663             );
6664         }
6665
6666         $fh->close();
6667     };
6668
6669     eval {
6670         # enable interrupts
6671         local $SIG{INT} =
6672             local $SIG{TERM} =
6673             local $SIG{QUIT} =
6674             local $SIG{HUP} =
6675             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6676         local $SIG{ALRM} = sub { die "got timeout\n"; };
6677
6678         $oldtimeout = alarm($timeout);
6679
6680         my $parser = sub {
6681             my $line = shift;
6682
6683             print "$line\n";
6684
6685             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6686                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6687                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6688             } elsif ($line =~ m/^CTIME: /) {
6689                 # we correctly received the vma config, so we can disable
6690                 # the timeout now for disk allocation (set to 10 minutes, so
6691                 # that we always timeout if something goes wrong)
6692                 alarm(600);
6693                 &$print_devmap();
6694                 print $fifofh "done\n";
6695                 my $tmp = $oldtimeout || 0;
6696                 $oldtimeout = undef;
6697                 alarm($tmp);
6698                 close($fifofh);
6699                 $fifofh = undef;
6700             }
6701         };
6702
6703         print "restore vma archive: $dbg_cmdstring\n";
6704         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6705     };
6706     my $err = $@;
6707
6708     alarm($oldtimeout) if $oldtimeout;
6709
6710     $restore_deactivate_volumes->($cfg, $devinfo);
6711
6712     close($fifofh) if $fifofh;
6713     unlink $mapfifo;
6714     rmtree $tmpdir;
6715
6716     if ($err) {
6717         $restore_destroy_volumes->($cfg, $devinfo);
6718         die $err;
6719     }
6720
6721     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6722
6723     PVE::Cluster::cfs_update(); # make sure we read new file
6724
6725     eval { rescan($vmid, 1); };
6726     warn $@ if $@;
6727
6728     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6729 }
6730
6731 sub restore_tar_archive {
6732     my ($archive, $vmid, $user, $opts) = @_;
6733
6734     if ($archive ne '-') {
6735         my $firstfile = tar_archive_read_firstfile($archive);
6736         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6737             if $firstfile ne 'qemu-server.conf';
6738     }
6739
6740     my $storecfg = PVE::Storage::config();
6741
6742     # avoid zombie disks when restoring over an existing VM -> cleanup first
6743     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6744     # skiplock=1 because qmrestore has set the 'create' lock itself already
6745     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6746     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6747
6748     my $tocmd = "/usr/lib/qemu-server/qmextract";
6749
6750     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6751     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6752     $tocmd .= ' --prealloc' if $opts->{prealloc};
6753     $tocmd .= ' --info' if $opts->{info};
6754
6755     # tar option "xf" does not autodetect compression when read from STDIN,
6756     # so we pipe to zcat
6757     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6758         PVE::Tools::shellquote("--to-command=$tocmd");
6759
6760     my $tmpdir = "/var/tmp/vzdumptmp$$";
6761     mkpath $tmpdir;
6762
6763     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6764     local $ENV{VZDUMP_VMID} = $vmid;
6765     local $ENV{VZDUMP_USER} = $user;
6766
6767     my $conffile = PVE::QemuConfig->config_file($vmid);
6768     my $new_conf_raw = '';
6769
6770     # disable interrupts (always do cleanups)
6771     local $SIG{INT} =
6772         local $SIG{TERM} =
6773         local $SIG{QUIT} =
6774         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6775
6776     eval {
6777         # enable interrupts
6778         local $SIG{INT} =
6779             local $SIG{TERM} =
6780             local $SIG{QUIT} =
6781             local $SIG{HUP} =
6782             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6783
6784         if ($archive eq '-') {
6785             print "extracting archive from STDIN\n";
6786             run_command($cmd, input => "<&STDIN");
6787         } else {
6788             print "extracting archive '$archive'\n";
6789             run_command($cmd);
6790         }
6791
6792         return if $opts->{info};
6793
6794         # read new mapping
6795         my $map = {};
6796         my $statfile = "$tmpdir/qmrestore.stat";
6797         if (my $fd = IO::File->new($statfile, "r")) {
6798             while (defined (my $line = <$fd>)) {
6799                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6800                     $map->{$1} = $2 if $1;
6801                 } else {
6802                     print STDERR "unable to parse line in statfile - $line\n";
6803                 }
6804             }
6805             $fd->close();
6806         }
6807
6808         my $confsrc = "$tmpdir/qemu-server.conf";
6809
6810         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6811
6812         my $cookie = { netcount => 0 };
6813         while (defined (my $line = <$srcfd>)) {
6814             $new_conf_raw .= restore_update_config_line(
6815                 $cookie,
6816                 $map,
6817                 $line,
6818                 $opts->{unique},
6819             );
6820         }
6821
6822         $srcfd->close();
6823     };
6824     if (my $err = $@) {
6825         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6826         die $err;
6827     }
6828
6829     rmtree $tmpdir;
6830
6831     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6832
6833     PVE::Cluster::cfs_update(); # make sure we read new file
6834
6835     eval { rescan($vmid, 1); };
6836     warn $@ if $@;
6837 };
6838
6839 sub foreach_storage_used_by_vm {
6840     my ($conf, $func) = @_;
6841
6842     my $sidhash = {};
6843
6844     PVE::QemuConfig->foreach_volume($conf, sub {
6845         my ($ds, $drive) = @_;
6846         return if drive_is_cdrom($drive);
6847
6848         my $volid = $drive->{file};
6849
6850         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6851         $sidhash->{$sid} = $sid if $sid;
6852     });
6853
6854     foreach my $sid (sort keys %$sidhash) {
6855         &$func($sid);
6856     }
6857 }
6858
6859 my $qemu_snap_storage = {
6860     rbd => 1,
6861 };
6862 sub do_snapshots_with_qemu {
6863     my ($storecfg, $volid) = @_;
6864
6865     my $storage_name = PVE::Storage::parse_volume_id($volid);
6866     my $scfg = $storecfg->{ids}->{$storage_name};
6867     die "could not find storage '$storage_name'\n" if !defined($scfg);
6868
6869     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6870         return 1;
6871     }
6872
6873     if ($volid =~ m/\.(qcow2|qed)$/){
6874         return 1;
6875     }
6876
6877     return;
6878 }
6879
6880 sub qga_check_running {
6881     my ($vmid, $nowarn) = @_;
6882
6883     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6884     if ($@) {
6885         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6886         return 0;
6887     }
6888     return 1;
6889 }
6890
6891 sub template_create {
6892     my ($vmid, $conf, $disk) = @_;
6893
6894     my $storecfg = PVE::Storage::config();
6895
6896     PVE::QemuConfig->foreach_volume($conf, sub {
6897         my ($ds, $drive) = @_;
6898
6899         return if drive_is_cdrom($drive);
6900         return if $disk && $ds ne $disk;
6901
6902         my $volid = $drive->{file};
6903         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6904
6905         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6906         $drive->{file} = $voliddst;
6907         $conf->{$ds} = print_drive($drive);
6908         PVE::QemuConfig->write_config($vmid, $conf);
6909     });
6910 }
6911
6912 sub convert_iscsi_path {
6913     my ($path) = @_;
6914
6915     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6916         my $portal = $1;
6917         my $target = $2;
6918         my $lun = $3;
6919
6920         my $initiator_name = get_initiator_name();
6921
6922         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6923                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6924     }
6925
6926     die "cannot convert iscsi path '$path', unkown format\n";
6927 }
6928
6929 sub qemu_img_convert {
6930     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6931
6932     my $storecfg = PVE::Storage::config();
6933     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6934     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6935
6936     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6937
6938     my $cachemode;
6939     my $src_path;
6940     my $src_is_iscsi = 0;
6941     my $src_format;
6942
6943     if ($src_storeid) {
6944         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6945         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6946         $src_format = qemu_img_format($src_scfg, $src_volname);
6947         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6948         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6949         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6950     } elsif (-f $src_volid) {
6951         $src_path = $src_volid;
6952         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6953             $src_format = $1;
6954         }
6955     }
6956
6957     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6958
6959     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6960     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6961     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6962     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6963
6964     my $cmd = [];
6965     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6966     push @$cmd, '-l', "snapshot.name=$snapname"
6967         if $snapname && $src_format && $src_format eq "qcow2";
6968     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6969     push @$cmd, '-T', $cachemode if defined($cachemode);
6970
6971     if ($src_is_iscsi) {
6972         push @$cmd, '--image-opts';
6973         $src_path = convert_iscsi_path($src_path);
6974     } elsif ($src_format) {
6975         push @$cmd, '-f', $src_format;
6976     }
6977
6978     if ($dst_is_iscsi) {
6979         push @$cmd, '--target-image-opts';
6980         $dst_path = convert_iscsi_path($dst_path);
6981     } else {
6982         push @$cmd, '-O', $dst_format;
6983     }
6984
6985     push @$cmd, $src_path;
6986
6987     if (!$dst_is_iscsi && $is_zero_initialized) {
6988         push @$cmd, "zeroinit:$dst_path";
6989     } else {
6990         push @$cmd, $dst_path;
6991     }
6992
6993     my $parser = sub {
6994         my $line = shift;
6995         if($line =~ m/\((\S+)\/100\%\)/){
6996             my $percent = $1;
6997             my $transferred = int($size * $percent / 100);
6998             my $total_h = render_bytes($size, 1);
6999             my $transferred_h = render_bytes($transferred, 1);
7000
7001             print "transferred $transferred_h of $total_h ($percent%)\n";
7002         }
7003
7004     };
7005
7006     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7007     my $err = $@;
7008     die "copy failed: $err" if $err;
7009 }
7010
7011 sub qemu_img_format {
7012     my ($scfg, $volname) = @_;
7013
7014     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7015         return $1;
7016     } else {
7017         return "raw";
7018     }
7019 }
7020
7021 sub qemu_drive_mirror {
7022     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7023
7024     $jobs = {} if !$jobs;
7025
7026     my $qemu_target;
7027     my $format;
7028     $jobs->{"drive-$drive"} = {};
7029
7030     if ($dst_volid =~ /^nbd:/) {
7031         $qemu_target = $dst_volid;
7032         $format = "nbd";
7033     } else {
7034         my $storecfg = PVE::Storage::config();
7035         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7036
7037         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7038
7039         $format = qemu_img_format($dst_scfg, $dst_volname);
7040
7041         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7042
7043         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7044     }
7045
7046     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7047     $opts->{format} = $format if $format;
7048
7049     if (defined($src_bitmap)) {
7050         $opts->{sync} = 'incremental';
7051         $opts->{bitmap} = $src_bitmap;
7052         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7053     }
7054
7055     if (defined($bwlimit)) {
7056         $opts->{speed} = $bwlimit * 1024;
7057         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7058     } else {
7059         print "drive mirror is starting for drive-$drive\n";
7060     }
7061
7062     # if a job already runs for this device we get an error, catch it for cleanup
7063     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7064     if (my $err = $@) {
7065         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7066         warn "$@\n" if $@;
7067         die "mirroring error: $err\n";
7068     }
7069
7070     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7071 }
7072
7073 # $completion can be either
7074 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7075 # 'cancel': wait until all jobs are ready, block-job-cancel them
7076 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7077 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7078 sub qemu_drive_mirror_monitor {
7079     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7080
7081     $completion //= 'complete';
7082     $op //= "mirror";
7083
7084     eval {
7085         my $err_complete = 0;
7086
7087         my $starttime = time ();
7088         while (1) {
7089             die "block job ('$op') timed out\n" if $err_complete > 300;
7090
7091             my $stats = mon_cmd($vmid, "query-block-jobs");
7092             my $ctime = time();
7093
7094             my $running_jobs = {};
7095             for my $stat (@$stats) {
7096                 next if $stat->{type} ne $op;
7097                 $running_jobs->{$stat->{device}} = $stat;
7098             }
7099
7100             my $readycounter = 0;
7101
7102             for my $job_id (sort keys %$jobs) {
7103                 my $job = $running_jobs->{$job_id};
7104
7105                 my $vanished = !defined($job);
7106                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7107                 if($complete || ($vanished && $completion eq 'auto')) {
7108                     print "$job_id: $op-job finished\n";
7109                     delete $jobs->{$job_id};
7110                     next;
7111                 }
7112
7113                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7114
7115                 my $busy = $job->{busy};
7116                 my $ready = $job->{ready};
7117                 if (my $total = $job->{len}) {
7118                     my $transferred = $job->{offset} || 0;
7119                     my $remaining = $total - $transferred;
7120                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7121
7122                     my $duration = $ctime - $starttime;
7123                     my $total_h = render_bytes($total, 1);
7124                     my $transferred_h = render_bytes($transferred, 1);
7125
7126                     my $status = sprintf(
7127                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7128                         render_duration($duration),
7129                     );
7130
7131                     if ($ready) {
7132                         if ($busy) {
7133                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7134                         } else {
7135                             $status .= ", ready";
7136                         }
7137                     }
7138                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7139                     $jobs->{$job_id}->{ready} = $ready;
7140                 }
7141
7142                 $readycounter++ if $job->{ready};
7143             }
7144
7145             last if scalar(keys %$jobs) == 0;
7146
7147             if ($readycounter == scalar(keys %$jobs)) {
7148                 print "all '$op' jobs are ready\n";
7149
7150                 # do the complete later (or has already been done)
7151                 last if $completion eq 'skip' || $completion eq 'auto';
7152
7153                 if ($vmiddst && $vmiddst != $vmid) {
7154                     my $agent_running = $qga && qga_check_running($vmid);
7155                     if ($agent_running) {
7156                         print "freeze filesystem\n";
7157                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7158                     } else {
7159                         print "suspend vm\n";
7160                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7161                     }
7162
7163                     # if we clone a disk for a new target vm, we don't switch the disk
7164                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7165
7166                     if ($agent_running) {
7167                         print "unfreeze filesystem\n";
7168                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7169                     } else {
7170                         print "resume vm\n";
7171                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7172                     }
7173
7174                     last;
7175                 } else {
7176
7177                     for my $job_id (sort keys %$jobs) {
7178                         # try to switch the disk if source and destination are on the same guest
7179                         print "$job_id: Completing block job_id...\n";
7180
7181                         my $op;
7182                         if ($completion eq 'complete') {
7183                             $op = 'block-job-complete';
7184                         } elsif ($completion eq 'cancel') {
7185                             $op = 'block-job-cancel';
7186                         } else {
7187                             die "invalid completion value: $completion\n";
7188                         }
7189                         eval { mon_cmd($vmid, $op, device => $job_id) };
7190                         if ($@ =~ m/cannot be completed/) {
7191                             print "$job_id: block job cannot be completed, trying again.\n";
7192                             $err_complete++;
7193                         }else {
7194                             print "$job_id: Completed successfully.\n";
7195                             $jobs->{$job_id}->{complete} = 1;
7196                         }
7197                     }
7198                 }
7199             }
7200             sleep 1;
7201         }
7202     };
7203     my $err = $@;
7204
7205     if ($err) {
7206         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7207         die "block job ($op) error: $err";
7208     }
7209 }
7210
7211 sub qemu_blockjobs_cancel {
7212     my ($vmid, $jobs) = @_;
7213
7214     foreach my $job (keys %$jobs) {
7215         print "$job: Cancelling block job\n";
7216         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7217         $jobs->{$job}->{cancel} = 1;
7218     }
7219
7220     while (1) {
7221         my $stats = mon_cmd($vmid, "query-block-jobs");
7222
7223         my $running_jobs = {};
7224         foreach my $stat (@$stats) {
7225             $running_jobs->{$stat->{device}} = $stat;
7226         }
7227
7228         foreach my $job (keys %$jobs) {
7229
7230             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7231                 print "$job: Done.\n";
7232                 delete $jobs->{$job};
7233             }
7234         }
7235
7236         last if scalar(keys %$jobs) == 0;
7237
7238         sleep 1;
7239     }
7240 }
7241
7242 sub clone_disk {
7243     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7244         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7245
7246     my $newvolid;
7247
7248     if (!$full) {
7249         print "create linked clone of drive $drivename ($drive->{file})\n";
7250         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7251         push @$newvollist, $newvolid;
7252     } else {
7253
7254         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7255         $storeid = $storage if $storage;
7256
7257         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7258
7259         print "create full clone of drive $drivename ($drive->{file})\n";
7260         my $name = undef;
7261         my $size = undef;
7262         if (drive_is_cloudinit($drive)) {
7263             $name = "vm-$newvmid-cloudinit";
7264             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7265             if ($scfg->{path}) {
7266                 $name .= ".$dst_format";
7267             }
7268             $snapname = undef;
7269             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7270         } elsif ($drivename eq 'efidisk0') {
7271             $size = get_efivars_size($conf);
7272         } else {
7273             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7274         }
7275         $newvolid = PVE::Storage::vdisk_alloc(
7276             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7277         );
7278         push @$newvollist, $newvolid;
7279
7280         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7281
7282         if (drive_is_cloudinit($drive)) {
7283             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7284             # if this is the case, we have to complete any block-jobs still there from
7285             # previous drive-mirrors
7286             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7287                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7288             }
7289             goto no_data_clone;
7290         }
7291
7292         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7293         if (!$running || $snapname) {
7294             # TODO: handle bwlimits
7295             if ($drivename eq 'efidisk0') {
7296                 # the relevant data on the efidisk may be smaller than the source
7297                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7298                 # that is given by the OVMF_VARS.fd
7299                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7300                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7301
7302                 # better for Ceph if block size is not too small, see bug #3324
7303                 my $bs = 1024*1024;
7304
7305                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7306                     "if=$src_path", "of=$dst_path"]);
7307             } else {
7308                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7309             }
7310         } else {
7311
7312             my $kvmver = get_running_qemu_version ($vmid);
7313             if (!min_version($kvmver, 2, 7)) {
7314                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7315                     if $drive->{iothread};
7316             }
7317
7318             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7319                 $completion, $qga, $bwlimit);
7320         }
7321     }
7322
7323 no_data_clone:
7324     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7325
7326     my $disk = $drive;
7327     $disk->{format} = undef;
7328     $disk->{file} = $newvolid;
7329     $disk->{size} = $size if defined($size);
7330
7331     return $disk;
7332 }
7333
7334 sub get_running_qemu_version {
7335     my ($vmid) = @_;
7336     my $res = mon_cmd($vmid, "query-version");
7337     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7338 }
7339
7340 sub qemu_use_old_bios_files {
7341     my ($machine_type) = @_;
7342
7343     return if !$machine_type;
7344
7345     my $use_old_bios_files = undef;
7346
7347     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7348         $machine_type = $1;
7349         $use_old_bios_files = 1;
7350     } else {
7351         my $version = extract_version($machine_type, kvm_user_version());
7352         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7353         # load new efi bios files on migration. So this hack is required to allow
7354         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7355         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7356         $use_old_bios_files = !min_version($version, 2, 4);
7357     }
7358
7359     return ($use_old_bios_files, $machine_type);
7360 }
7361
7362 sub get_efivars_size {
7363     my ($conf) = @_;
7364     my $arch = get_vm_arch($conf);
7365     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7366     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7367     return -s $ovmf_vars;
7368 }
7369
7370 sub update_efidisk_size {
7371     my ($conf) = @_;
7372
7373     return if !defined($conf->{efidisk0});
7374
7375     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7376     $disk->{size} = get_efivars_size($conf);
7377     $conf->{efidisk0} = print_drive($disk);
7378
7379     return;
7380 }
7381
7382 sub create_efidisk($$$$$) {
7383     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7384
7385     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7386     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7387
7388     my $vars_size_b = -s $ovmf_vars;
7389     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7390     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7391     PVE::Storage::activate_volumes($storecfg, [$volid]);
7392
7393     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7394     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7395
7396     return ($volid, $size/1024);
7397 }
7398
7399 sub vm_iothreads_list {
7400     my ($vmid) = @_;
7401
7402     my $res = mon_cmd($vmid, 'query-iothreads');
7403
7404     my $iothreads = {};
7405     foreach my $iothread (@$res) {
7406         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7407     }
7408
7409     return $iothreads;
7410 }
7411
7412 sub scsihw_infos {
7413     my ($conf, $drive) = @_;
7414
7415     my $maxdev = 0;
7416
7417     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7418         $maxdev = 7;
7419     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7420         $maxdev = 1;
7421     } else {
7422         $maxdev = 256;
7423     }
7424
7425     my $controller = int($drive->{index} / $maxdev);
7426     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7427         ? "virtioscsi"
7428         : "scsihw";
7429
7430     return ($maxdev, $controller, $controller_prefix);
7431 }
7432
7433 sub windows_version {
7434     my ($ostype) = @_;
7435
7436     return 0 if !$ostype;
7437
7438     my $winversion = 0;
7439
7440     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7441         $winversion = 5;
7442     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7443         $winversion = 6;
7444     } elsif ($ostype =~ m/^win(\d+)$/) {
7445         $winversion = $1;
7446     }
7447
7448     return $winversion;
7449 }
7450
7451 sub resolve_dst_disk_format {
7452         my ($storecfg, $storeid, $src_volname, $format) = @_;
7453         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7454
7455         if (!$format) {
7456             # if no target format is specified, use the source disk format as hint
7457             if ($src_volname) {
7458                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7459                 $format = qemu_img_format($scfg, $src_volname);
7460             } else {
7461                 return $defFormat;
7462             }
7463         }
7464
7465         # test if requested format is supported - else use default
7466         my $supported = grep { $_ eq $format } @$validFormats;
7467         $format = $defFormat if !$supported;
7468         return $format;
7469 }
7470
7471 # NOTE: if this logic changes, please update docs & possibly gui logic
7472 sub find_vmstate_storage {
7473     my ($conf, $storecfg) = @_;
7474
7475     # first, return storage from conf if set
7476     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7477
7478     my ($target, $shared, $local);
7479
7480     foreach_storage_used_by_vm($conf, sub {
7481         my ($sid) = @_;
7482         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7483         my $dst = $scfg->{shared} ? \$shared : \$local;
7484         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7485     });
7486
7487     # second, use shared storage where VM has at least one disk
7488     # third, use local storage where VM has at least one disk
7489     # fall back to local storage
7490     $target = $shared // $local // 'local';
7491
7492     return $target;
7493 }
7494
7495 sub generate_uuid {
7496     my ($uuid, $uuid_str);
7497     UUID::generate($uuid);
7498     UUID::unparse($uuid, $uuid_str);
7499     return $uuid_str;
7500 }
7501
7502 sub generate_smbios1_uuid {
7503     return "uuid=".generate_uuid();
7504 }
7505
7506 sub nbd_stop {
7507     my ($vmid) = @_;
7508
7509     mon_cmd($vmid, 'nbd-server-stop');
7510 }
7511
7512 sub create_reboot_request {
7513     my ($vmid) = @_;
7514     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7515         or die "failed to create reboot trigger file: $!\n";
7516     close($fh);
7517 }
7518
7519 sub clear_reboot_request {
7520     my ($vmid) = @_;
7521     my $path = "/run/qemu-server/$vmid.reboot";
7522     my $res = 0;
7523
7524     $res = unlink($path);
7525     die "could not remove reboot request for $vmid: $!"
7526         if !$res && $! != POSIX::ENOENT;
7527
7528     return $res;
7529 }
7530
7531 sub bootorder_from_legacy {
7532     my ($conf, $bootcfg) = @_;
7533
7534     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7535     my $bootindex_hash = {};
7536     my $i = 1;
7537     foreach my $o (split(//, $boot)) {
7538         $bootindex_hash->{$o} = $i*100;
7539         $i++;
7540     }
7541
7542     my $bootorder = {};
7543
7544     PVE::QemuConfig->foreach_volume($conf, sub {
7545         my ($ds, $drive) = @_;
7546
7547         if (drive_is_cdrom ($drive, 1)) {
7548             if ($bootindex_hash->{d}) {
7549                 $bootorder->{$ds} = $bootindex_hash->{d};
7550                 $bootindex_hash->{d} += 1;
7551             }
7552         } elsif ($bootindex_hash->{c}) {
7553             $bootorder->{$ds} = $bootindex_hash->{c}
7554                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7555             $bootindex_hash->{c} += 1;
7556         }
7557     });
7558
7559     if ($bootindex_hash->{n}) {
7560         for (my $i = 0; $i < $MAX_NETS; $i++) {
7561             my $netname = "net$i";
7562             next if !$conf->{$netname};
7563             $bootorder->{$netname} = $bootindex_hash->{n};
7564             $bootindex_hash->{n} += 1;
7565         }
7566     }
7567
7568     return $bootorder;
7569 }
7570
7571 # Generate default device list for 'boot: order=' property. Matches legacy
7572 # default boot order, but with explicit device names. This is important, since
7573 # the fallback for when neither 'order' nor the old format is specified relies
7574 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7575 sub get_default_bootdevices {
7576     my ($conf) = @_;
7577
7578     my @ret = ();
7579
7580     # harddisk
7581     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7582     push @ret, $first if $first;
7583
7584     # cdrom
7585     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7586     push @ret, $first if $first;
7587
7588     # network
7589     for (my $i = 0; $i < $MAX_NETS; $i++) {
7590         my $netname = "net$i";
7591         next if !$conf->{$netname};
7592         push @ret, $netname;
7593         last;
7594     }
7595
7596     return \@ret;
7597 }
7598
7599 sub device_bootorder {
7600     my ($conf) = @_;
7601
7602     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7603
7604     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7605
7606     my $bootorder = {};
7607     if (!defined($boot) || $boot->{legacy}) {
7608         $bootorder = bootorder_from_legacy($conf, $boot);
7609     } elsif ($boot->{order}) {
7610         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7611         for my $dev (PVE::Tools::split_list($boot->{order})) {
7612             $bootorder->{$dev} = $i++;
7613         }
7614     }
7615
7616     return $bootorder;
7617 }
7618
7619 sub register_qmeventd_handle {
7620     my ($vmid) = @_;
7621
7622     my $fh;
7623     my $peer = "/var/run/qmeventd.sock";
7624     my $count = 0;
7625
7626     for (;;) {
7627         $count++;
7628         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7629         last if $fh;
7630         if ($! != EINTR && $! != EAGAIN) {
7631             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7632         }
7633         if ($count > 4) {
7634             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7635               . "after $count retries\n";
7636         }
7637         usleep(25000);
7638     }
7639
7640     # send handshake to mark VM as backing up
7641     print $fh to_json({vzdump => {vmid => "$vmid"}});
7642
7643     # return handle to be closed later when inhibit is no longer required
7644     return $fh;
7645 }
7646
7647 # bash completion helper
7648
7649 sub complete_backup_archives {
7650     my ($cmdname, $pname, $cvalue) = @_;
7651
7652     my $cfg = PVE::Storage::config();
7653
7654     my $storeid;
7655
7656     if ($cvalue =~ m/^([^:]+):/) {
7657         $storeid = $1;
7658     }
7659
7660     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7661
7662     my $res = [];
7663     foreach my $id (keys %$data) {
7664         foreach my $item (@{$data->{$id}}) {
7665             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7666             push @$res, $item->{volid} if defined($item->{volid});
7667         }
7668     }
7669
7670     return $res;
7671 }
7672
7673 my $complete_vmid_full = sub {
7674     my ($running) = @_;
7675
7676     my $idlist = vmstatus();
7677
7678     my $res = [];
7679
7680     foreach my $id (keys %$idlist) {
7681         my $d = $idlist->{$id};
7682         if (defined($running)) {
7683             next if $d->{template};
7684             next if $running && $d->{status} ne 'running';
7685             next if !$running && $d->{status} eq 'running';
7686         }
7687         push @$res, $id;
7688
7689     }
7690     return $res;
7691 };
7692
7693 sub complete_vmid {
7694     return &$complete_vmid_full();
7695 }
7696
7697 sub complete_vmid_stopped {
7698     return &$complete_vmid_full(0);
7699 }
7700
7701 sub complete_vmid_running {
7702     return &$complete_vmid_full(1);
7703 }
7704
7705 sub complete_storage {
7706
7707     my $cfg = PVE::Storage::config();
7708     my $ids = $cfg->{ids};
7709
7710     my $res = [];
7711     foreach my $sid (keys %$ids) {
7712         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7713         next if !$ids->{$sid}->{content}->{images};
7714         push @$res, $sid;
7715     }
7716
7717     return $res;
7718 }
7719
7720 sub complete_migration_storage {
7721     my ($cmd, $param, $current_value, $all_args) = @_;
7722
7723     my $targetnode = @$all_args[1];
7724
7725     my $cfg = PVE::Storage::config();
7726     my $ids = $cfg->{ids};
7727
7728     my $res = [];
7729     foreach my $sid (keys %$ids) {
7730         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7731         next if !$ids->{$sid}->{content}->{images};
7732         push @$res, $sid;
7733     }
7734
7735     return $res;
7736 }
7737
7738 sub vm_is_paused {
7739     my ($vmid) = @_;
7740     my $qmpstatus = eval {
7741         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7742         mon_cmd($vmid, "query-status");
7743     };
7744     warn "$@\n" if $@;
7745     return $qmpstatus && $qmpstatus->{status} eq "paused";
7746 }
7747
7748 sub check_volume_storage_type {
7749     my ($storecfg, $vol) = @_;
7750
7751     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7752     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7753     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7754
7755     die "storage '$storeid' does not support content-type '$vtype'\n"
7756         if !$scfg->{content}->{$vtype};
7757
7758     return 1;
7759 }
7760
7761 1;