PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Shown in the web-interface VM's summary."
 369             ." This is saved as comment inside the configuration file.",
 370         maxLength => 1024 * 8,
 371     },
 372     ostype => {
 373         optional => 1,
 374         type => 'string',
 375         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 376         description => "Specify guest operating system.",
 377         verbose_description => <<EODESC,
 378 Specify guest operating system. This is used to enable special
 379 optimization/features for specific operating systems:
 380
 381 [horizontal]
 382 other;; unspecified OS
 383 wxp;; Microsoft Windows XP
 384 w2k;; Microsoft Windows 2000
 385 w2k3;; Microsoft Windows 2003
 386 w2k8;; Microsoft Windows 2008
 387 wvista;; Microsoft Windows Vista
 388 win7;; Microsoft Windows 7
 389 win8;; Microsoft Windows 8/2012/2012r2
 390 win10;; Microsoft Windows 10/2016/2019
 391 l24;; Linux 2.4 Kernel
 392 l26;; Linux 2.6 - 5.X Kernel
 393 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 394 EODESC
 395     },
 396     boot => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-boot',
 399         description => "Specify guest boot order. Use with 'order=', usage with"
 400                      . " no key or 'legacy=' is deprecated.",
 401     },
 402     bootdisk => {
 403         optional => 1,
 404         type => 'string', format => 'pve-qm-bootdisk',
 405         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 406         pattern => '(ide|sata|scsi|virtio)\d+',
 407     },
 408     smp => {
 409         optional => 1,
 410         type => 'integer',
 411         description => "The number of CPUs. Please use option -sockets instead.",
 412         minimum => 1,
 413         default => 1,
 414     },
 415     sockets => {
 416         optional => 1,
 417         type => 'integer',
 418         description => "The number of CPU sockets.",
 419         minimum => 1,
 420         default => 1,
 421     },
 422     cores => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "The number of cores per socket.",
 426         minimum => 1,
 427         default => 1,
 428     },
 429     numa => {
 430         optional => 1,
 431         type => 'boolean',
 432         description => "Enable/disable NUMA.",
 433         default => 0,
 434     },
 435     hugepages => {
 436         optional => 1,
 437         type => 'string',
 438         description => "Enable/disable hugepages memory.",
 439         enum => [qw(any 2 1024)],
 440     },
 441     keephugepages => {
 442         optional => 1,
 443         type => 'boolean',
 444         default => 0,
 445         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 446             ." after VM shutdown and can be used for subsequent starts.",
 447     },
 448     vcpus => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "Number of hotplugged vcpus.",
 452         minimum => 1,
 453         default => 0,
 454     },
 455     acpi => {
 456         optional => 1,
 457         type => 'boolean',
 458         description => "Enable/disable ACPI.",
 459         default => 1,
 460     },
 461     agent => {
 462         optional => 1,
 463         description => "Enable/disable Qemu GuestAgent and its properties.",
 464         type => 'string',
 465         format => $agent_fmt,
 466     },
 467     kvm => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable KVM hardware virtualization.",
 471         default => 1,
 472     },
 473     tdf => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable time drift fix.",
 477         default => 0,
 478     },
 479     localtime => {
 480         optional => 1,
 481         type => 'boolean',
 482         description => "Set the real time clock to local time. This is enabled by default if ostype"
 483             ." indicates a Microsoft OS.",
 484     },
 485     freeze => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 489     },
 490     vga => {
 491         optional => 1,
 492         type => 'string', format => $vga_fmt,
 493         description => "Configure the VGA hardware.",
 494         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 495             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 496             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 497             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 498             ." display server. For win* OS you can select how many independent displays you want,"
 499             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 500             ." using a serial device as terminal.",
 501     },
 502     watchdog => {
 503         optional => 1,
 504         type => 'string', format => 'pve-qm-watchdog',
 505         description => "Create a virtual hardware watchdog device.",
 506         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 507             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 508             ." else the watchdog will reset the guest (or execute the respective action specified)",
 509     },
 510     startdate => {
 511         optional => 1,
 512         type => 'string',
 513         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 514         description => "Set the initial date of the real time clock. Valid format for date are:"
 515             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 516         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 517         default => 'now',
 518     },
 519     startup =>  get_standard_option('pve-startup-order'),
 520     template => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Enable/disable Template.",
 524         default => 0,
 525     },
 526     args => {
 527         optional => 1,
 528         type => 'string',
 529         description => "Arbitrary arguments passed to kvm.",
 530         verbose_description => <<EODESCR,
 531 Arbitrary arguments passed to kvm, for example:
 532
 533 args: -no-reboot -no-hpet
 534
 535 NOTE: this option is for experts only.
 536 EODESCR
 537     },
 538     tablet => {
 539         optional => 1,
 540         type => 'boolean',
 541         default => 1,
 542         description => "Enable/disable the USB tablet device.",
 543         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 544             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 545             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 546             ." may consider disabling this to save some context switches. This is turned off by"
 547             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 548     },
 549     migrate_speed => {
 550         optional => 1,
 551         type => 'integer',
 552         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 553         minimum => 0,
 554         default => 0,
 555     },
 556     migrate_downtime => {
 557         optional => 1,
 558         type => 'number',
 559         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 560         minimum => 0,
 561         default => 0.1,
 562     },
 563     cdrom => {
 564         optional => 1,
 565         type => 'string', format => 'pve-qm-ide',
 566         typetext => '<volume>',
 567         description => "This is an alias for option -ide2",
 568     },
 569     cpu => {
 570         optional => 1,
 571         description => "Emulated CPU type.",
 572         type => 'string',
 573         format => 'pve-vm-cpu-conf',
 574     },
 575     parent => get_standard_option('pve-snapshot-name', {
 576         optional => 1,
 577         description => "Parent snapshot name. This is used internally, and should not be modified.",
 578     }),
 579     snaptime => {
 580         optional => 1,
 581         description => "Timestamp for snapshots.",
 582         type => 'integer',
 583         minimum => 0,
 584     },
 585     vmstate => {
 586         optional => 1,
 587         type => 'string', format => 'pve-volume-id',
 588         description => "Reference to a volume which stores the VM state. This is used internally"
 589             ." for snapshots.",
 590     },
 591     vmstatestorage => get_standard_option('pve-storage-id', {
 592         description => "Default storage for VM state volumes/files.",
 593         optional => 1,
 594     }),
 595     runningmachine => get_standard_option('pve-qemu-machine', {
 596         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 597             ." for snapshots.",
 598     }),
 599     runningcpu => {
 600         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 601             ." internally for snapshots.",
 602         optional => 1,
 603         type => 'string',
 604         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 605         format_description => 'QEMU -cpu parameter'
 606     },
 607     machine => get_standard_option('pve-qemu-machine'),
 608     arch => {
 609         description => "Virtual processor architecture. Defaults to the host.",
 610         optional => 1,
 611         type => 'string',
 612         enum => [qw(x86_64 aarch64)],
 613     },
 614     smbios1 => {
 615         description => "Specify SMBIOS type 1 fields.",
 616         type => 'string', format => 'pve-qm-smbios1',
 617         maxLength => 512,
 618         optional => 1,
 619     },
 620     protection => {
 621         optional => 1,
 622         type => 'boolean',
 623         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 624             ." remove disk operations.",
 625         default => 0,
 626     },
 627     bios => {
 628         optional => 1,
 629         type => 'string',
 630         enum => [ qw(seabios ovmf) ],
 631         description => "Select BIOS implementation.",
 632         default => 'seabios',
 633     },
 634     vmgenid => {
 635         type => 'string',
 636         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 637         format_description => 'UUID',
 638         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 639             ." to disable explicitly.",
 640         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 641             ." value identifier to the guest OS. This allows to notify the guest operating system"
 642             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 643             ." execution or creation from a template). The guest operating system notices the"
 644             ." change, and is then able to react as appropriate by marking its copies of"
 645             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 646             ."Note that auto-creation only works when done through API/CLI create or update methods"
 647             .", but not when manually editing the config file.",
 648         default => "1 (autogenerated)",
 649         optional => 1,
 650     },
 651     hookscript => {
 652         type => 'string',
 653         format => 'pve-volume-id',
 654         optional => 1,
 655         description => "Script that will be executed during various steps in the vms lifetime.",
 656     },
 657     ivshmem => {
 658         type => 'string',
 659         format => $ivshmem_fmt,
 660         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 661             ." the host.",
 662         optional => 1,
 663     },
 664     audio0 => {
 665         type => 'string',
 666         format => $audio_fmt,
 667         description => "Configure a audio device, useful in combination with QXL/Spice.",
 668         optional => 1
 669     },
 670     spice_enhancements => {
 671         type => 'string',
 672         format => $spice_enhancements_fmt,
 673         description => "Configure additional enhancements for SPICE.",
 674         optional => 1
 675     },
 676     tags => {
 677         type => 'string', format => 'pve-tag-list',
 678         description => 'Tags of the VM. This is only meta information.',
 679         optional => 1,
 680     },
 681     rng0 => {
 682         type => 'string',
 683         format => $rng_fmt,
 684         description => "Configure a VirtIO-based Random Number Generator.",
 685         optional => 1,
 686     },
 687 };
 688
 689 my $cicustom_fmt = {
 690     meta => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all meta data passed to the VM via"
 694             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 695         format => 'pve-volume-id',
 696         format_description => 'volume',
 697     },
 698     network => {
 699         type => 'string',
 700         optional => 1,
 701         description => 'Specify a custom file containing all network data passed to the VM via'
 702             .' cloud-init.',
 703         format => 'pve-volume-id',
 704         format_description => 'volume',
 705     },
 706     user => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all user data passed to the VM via'
 710             .' cloud-init.',
 711         format => 'pve-volume-id',
 712         format_description => 'volume',
 713     },
 714 };
 715 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 716
 717 my $confdesc_cloudinit = {
 718     citype => {
 719         optional => 1,
 720         type => 'string',
 721         description => 'Specifies the cloud-init configuration format. The default depends on the'
 722             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 723             .' and `configdrive2` for windows.',
 724         enum => ['configdrive2', 'nocloud', 'opennebula'],
 725     },
 726     ciuser => {
 727         optional => 1,
 728         type => 'string',
 729         description => "cloud-init: User name to change ssh keys and password for instead of the"
 730             ." image's configured default user.",
 731     },
 732     cipassword => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'cloud-init: Password to assign the user. Using this is generally not'
 736             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 737             .' support hashed passwords.',
 738     },
 739     cicustom => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'cloud-init: Specify custom files to replace the automatically generated'
 743             .' ones at start.',
 744         format => 'pve-qm-cicustom',
 745     },
 746     searchdomain => {
 747         optional => 1,
 748         type => 'string',
 749         description => "cloud-init: Sets DNS search domains for a container. Create will'
 750             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 751             .' are set.",
 752     },
 753     nameserver => {
 754         optional => 1,
 755         type => 'string', format => 'address-list',
 756         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 757             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 758             .' are set.",
 759     },
 760     sshkeys => {
 761         optional => 1,
 762         type => 'string',
 763         format => 'urlencoded',
 764         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 765     },
 766 };
 767
 768 # what about other qemu settings ?
 769 #cpu => 'string',
 770 #machine => 'string',
 771 #fda => 'file',
 772 #fdb => 'file',
 773 #mtdblock => 'file',
 774 #sd => 'file',
 775 #pflash => 'file',
 776 #snapshot => 'bool',
 777 #bootp => 'file',
 778 ##tftp => 'dir',
 779 ##smb => 'dir',
 780 #kernel => 'file',
 781 #append => 'string',
 782 #initrd => 'file',
 783 ##soundhw => 'string',
 784
 785 while (my ($k, $v) = each %$confdesc) {
 786     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 787 }
 788
 789 my $MAX_USB_DEVICES = 5;
 790 my $MAX_NETS = 32;
 791 my $MAX_SERIAL_PORTS = 4;
 792 my $MAX_PARALLEL_PORTS = 3;
 793 my $MAX_NUMA = 8;
 794
 795 my $numa_fmt = {
 796     cpus => {
 797         type => "string",
 798         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 799         description => "CPUs accessing this NUMA node.",
 800         format_description => "id[-id];...",
 801     },
 802     memory => {
 803         type => "number",
 804         description => "Amount of memory this NUMA node provides.",
 805         optional => 1,
 806     },
 807     hostnodes => {
 808         type => "string",
 809         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 810         description => "Host NUMA nodes to use.",
 811         format_description => "id[-id];...",
 812         optional => 1,
 813     },
 814     policy => {
 815         type => 'string',
 816         enum => [qw(preferred bind interleave)],
 817         description => "NUMA allocation policy.",
 818         optional => 1,
 819     },
 820 };
 821 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 822 my $numadesc = {
 823     optional => 1,
 824     type => 'string', format => $numa_fmt,
 825     description => "NUMA topology.",
 826 };
 827 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 828
 829 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 830     $confdesc->{"numa$i"} = $numadesc;
 831 }
 832
 833 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 834                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 835                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 836 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 837
 838 my $net_fmt_bridge_descr = <<__EOD__;
 839 Bridge to attach the network device to. The Proxmox VE standard bridge
 840 is called 'vmbr0'.
 841
 842 If you do not specify a bridge, we create a kvm user (NATed) network
 843 device, which provides DHCP and DNS services. The following addresses
 844 are used:
 845
 846  10.0.2.2   Gateway
 847  10.0.2.3   DNS Server
 848  10.0.2.4   SMB Server
 849
 850 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 851 __EOD__
 852
 853 my $net_fmt = {
 854     macaddr  => get_standard_option('mac-addr', {
 855         description => "MAC address. That address must be unique withing your network. This is"
 856             ." automatically generated if not specified.",
 857     }),
 858     model => {
 859         type => 'string',
 860         description => "Network Card Model. The 'virtio' model provides the best performance with"
 861             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 862             ." best to use 'e1000'.",
 863         enum => $nic_model_list,
 864         default_key => 1,
 865     },
 866     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 867     bridge => {
 868         type => 'string',
 869         description => $net_fmt_bridge_descr,
 870         format_description => 'bridge',
 871         pattern => '[-_.\w\d]+',
 872         optional => 1,
 873     },
 874     queues => {
 875         type => 'integer',
 876         minimum => 0, maximum => 16,
 877         description => 'Number of packet queues to be used on the device.',
 878         optional => 1,
 879     },
 880     rate => {
 881         type => 'number',
 882         minimum => 0,
 883         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 884         optional => 1,
 885     },
 886     tag => {
 887         type => 'integer',
 888         minimum => 1, maximum => 4094,
 889         description => 'VLAN tag to apply to packets on this interface.',
 890         optional => 1,
 891     },
 892     trunks => {
 893         type => 'string',
 894         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 895         description => 'VLAN trunks to pass through this interface.',
 896         format_description => 'vlanid[;vlanid...]',
 897         optional => 1,
 898     },
 899     firewall => {
 900         type => 'boolean',
 901         description => 'Whether this interface should be protected by the firewall.',
 902         optional => 1,
 903     },
 904     link_down => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be disconnected (like pulling the plug).',
 907         optional => 1,
 908     },
 909     mtu => {
 910         type => 'integer',
 911         minimum => 1, maximum => 65520,
 912         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 913         optional => 1,
 914     },
 915 };
 916
 917 my $netdesc = {
 918     optional => 1,
 919     type => 'string', format => $net_fmt,
 920     description => "Specify network devices.",
 921 };
 922
 923 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 924
 925 my $ipconfig_fmt = {
 926     ip => {
 927         type => 'string',
 928         format => 'pve-ipv4-config',
 929         format_description => 'IPv4Format/CIDR',
 930         description => 'IPv4 address in CIDR format.',
 931         optional => 1,
 932         default => 'dhcp',
 933     },
 934     gw => {
 935         type => 'string',
 936         format => 'ipv4',
 937         format_description => 'GatewayIPv4',
 938         description => 'Default gateway for IPv4 traffic.',
 939         optional => 1,
 940         requires => 'ip',
 941     },
 942     ip6 => {
 943         type => 'string',
 944         format => 'pve-ipv6-config',
 945         format_description => 'IPv6Format/CIDR',
 946         description => 'IPv6 address in CIDR format.',
 947         optional => 1,
 948         default => 'dhcp',
 949     },
 950     gw6 => {
 951         type => 'string',
 952         format => 'ipv6',
 953         format_description => 'GatewayIPv6',
 954         description => 'Default gateway for IPv6 traffic.',
 955         optional => 1,
 956         requires => 'ip6',
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 960 my $ipconfigdesc = {
 961     optional => 1,
 962     type => 'string', format => 'pve-qm-ipconfig',
 963     description => <<'EODESCR',
 964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 965
 966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 967
 968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 969 gateway should be provided.
 970 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 971 cloud-init 19.4 or newer.
 972
 973 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 974 dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $usb_fmt = {
1006     host => {
1007         default_key => 1,
1008         type => 'string', format => 'pve-qm-usb-device',
1009         format_description => 'HOSTUSBDEVICE|spice',
1010         description => <<EODESCR,
1011 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1012
1013  'bus-port(.port)*' (decimal numbers) or
1014  'vendor_id:product_id' (hexadeciaml numbers) or
1015  'spice'
1016
1017 You can use the 'lsusb -t' command to list existing usb devices.
1018
1019 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1020 machines - use with special care.
1021
1022 The value 'spice' can be used to add a usb redirection devices for spice.
1023 EODESCR
1024     },
1025     usb3 => {
1026         optional => 1,
1027         type => 'boolean',
1028         description => "Specifies whether if given host option is a USB3 device or port.",
1029         default => 0,
1030     },
1031 };
1032
1033 my $usbdesc = {
1034     optional => 1,
1035     type => 'string', format => $usb_fmt,
1036     description => "Configure an USB device (n is 0 to 4).",
1037 };
1038 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1039
1040 my $serialdesc = {
1041         optional => 1,
1042         type => 'string',
1043         pattern => '(/dev/.+|socket)',
1044         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1045         verbose_description =>  <<EODESCR,
1046 Create a serial device inside the VM (n is 0 to 3), and pass through a
1047 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1048 host side (use 'qm terminal' to open a terminal connection).
1049
1050 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1051 use with special care.
1052
1053 CAUTION: Experimental! User reported problems with this option.
1054 EODESCR
1055 };
1056
1057 my $paralleldesc= {
1058         optional => 1,
1059         type => 'string',
1060         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1061         description =>  "Map host parallel devices (n is 0 to 2).",
1062         verbose_description =>  <<EODESCR,
1063 Map host parallel devices (n is 0 to 2).
1064
1065 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1066 machines - use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1073     $confdesc->{"parallel$i"} = $paralleldesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1077     $confdesc->{"serial$i"} = $serialdesc;
1078 }
1079
1080 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1081     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1082 }
1083
1084 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1085     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1086 }
1087
1088 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1089     $confdesc->{"usb$i"} = $usbdesc;
1090 }
1091
1092 my $boot_fmt = {
1093     legacy => {
1094         optional => 1,
1095         default_key => 1,
1096         type => 'string',
1097         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1098                      . " Deprecated, use 'order=' instead.",
1099         pattern => '[acdn]{1,4}',
1100         format_description => "[acdn]{1,4}",
1101
1102         # note: this is also the fallback if boot: is not given at all
1103         default => 'cdn',
1104     },
1105     order => {
1106         optional => 1,
1107         type => 'string',
1108         format => 'pve-qm-bootdev-list',
1109         format_description => "device[;device...]",
1110         description => <<EODESC,
1111 The guest will attempt to boot from devices in the order they appear here.
1112
1113 Disks, optical drives and passed-through storage USB devices will be directly
1114 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1115 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1116
1117 Note that only devices in this list will be marked as bootable and thus loaded
1118 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1119 (e.g. software-raid), you need to specify all of them here.
1120
1121 Overrides the deprecated 'legacy=[acdn]*' value when given.
1122 EODESC
1123     },
1124 };
1125 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1126
1127 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1128 sub verify_bootdev {
1129     my ($dev, $noerr) = @_;
1130
1131     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1132
1133     my $check = sub {
1134         my ($base) = @_;
1135         return 0 if $dev !~ m/^$base\d+$/;
1136         return 0 if !$confdesc->{$dev};
1137         return 1;
1138     };
1139
1140     return $dev if $check->("net");
1141     return $dev if $check->("usb");
1142     return $dev if $check->("hostpci");
1143
1144     return if $noerr;
1145     die "invalid boot device '$dev'\n";
1146 }
1147
1148 sub print_bootorder {
1149     my ($devs) = @_;
1150     return "" if !@$devs;
1151     my $data = { order => join(';', @$devs) };
1152     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1153 }
1154
1155 my $kvm_api_version = 0;
1156
1157 sub kvm_version {
1158     return $kvm_api_version if $kvm_api_version;
1159
1160     open my $fh, '<', '/dev/kvm' or return;
1161
1162     # 0xae00 => KVM_GET_API_VERSION
1163     $kvm_api_version = ioctl($fh, 0xae00, 0);
1164     close($fh);
1165
1166     return $kvm_api_version;
1167 }
1168
1169 my $kvm_user_version = {};
1170 my $kvm_mtime = {};
1171
1172 sub kvm_user_version {
1173     my ($binary) = @_;
1174
1175     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1176     my $st = stat($binary);
1177
1178     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180         $cachedmtime == $st->mtime;
1181
1182     $kvm_user_version->{$binary} = 'unknown';
1183     $kvm_mtime->{$binary} = $st->mtime;
1184
1185     my $code = sub {
1186         my $line = shift;
1187         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188             $kvm_user_version->{$binary} = $2;
1189         }
1190     };
1191
1192     eval { run_command([$binary, '--version'], outfunc => $code); };
1193     warn $@ if $@;
1194
1195     return $kvm_user_version->{$binary};
1196
1197 }
1198 my sub extract_version {
1199     my ($machine_type, $version) = @_;
1200     $version = kvm_user_version() if !defined($version);
1201     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1202 }
1203
1204 sub kernel_has_vhost_net {
1205     return -c '/dev/vhost-net';
1206 }
1207
1208 sub option_exists {
1209     my $key = shift;
1210     return defined($confdesc->{$key});
1211 }
1212
1213 my $cdrom_path;
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241      if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1286             if !$vtype;
1287         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1288         verify_media_type($opt, $vtype, $drive->{media});
1289         $drive->{file} = $volid;
1290     }
1291
1292     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1293 }
1294
1295 sub parse_hotplug_features {
1296     my ($data) = @_;
1297
1298     my $res = {};
1299
1300     return $res if $data eq '0';
1301
1302     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1303
1304     foreach my $feature (PVE::Tools::split_list($data)) {
1305         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306             $res->{$1} = 1;
1307         } else {
1308             die "invalid hotplug feature '$feature'\n";
1309         }
1310     }
1311     return $res;
1312 }
1313
1314 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1315 sub pve_verify_hotplug_features {
1316     my ($value, $noerr) = @_;
1317
1318     return $value if parse_hotplug_features($value);
1319
1320     return if $noerr;
1321
1322     die "unable to parse hotplug option\n";
1323 }
1324
1325 sub scsi_inquiry {
1326     my($fh, $noerr) = @_;
1327
1328     my $SG_IO = 0x2285;
1329     my $SG_GET_VERSION_NUM = 0x2282;
1330
1331     my $versionbuf = "\x00" x 8;
1332     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333     if (!$ret) {
1334         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335         return;
1336     }
1337     my $version = unpack("I", $versionbuf);
1338     if ($version < 30000) {
1339         die "scsi generic interface too old\n"  if !$noerr;
1340         return;
1341     }
1342
1343     my $buf = "\x00" x 36;
1344     my $sensebuf = "\x00" x 8;
1345     my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347     # see /usr/include/scsi/sg.h
1348     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351                       length($sensebuf), 0, length($buf), $buf,
1352                       $cmd, $sensebuf, 6000);
1353
1354     $ret = ioctl($fh, $SG_IO, $packet);
1355     if (!$ret) {
1356         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357         return;
1358     }
1359
1360     my @res = unpack($sg_io_hdr_t, $packet);
1361     if ($res[17] || $res[18]) {
1362         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363         return;
1364     }
1365
1366     my $res = {};
1367     (my $byte0, my $byte1, $res->{vendor},
1368      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370     $res->{removable} = $byte1 & 128 ? 1 : 0;
1371     $res->{type} = $byte0 & 31;
1372
1373     return $res;
1374 }
1375
1376 sub path_is_scsi {
1377     my ($path) = @_;
1378
1379     my $fh = IO::File->new("+<$path") || return;
1380     my $res = scsi_inquiry($fh, 1);
1381     close($fh);
1382
1383     return $res;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf, $arch) = @_;
1388
1389     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus;
1393     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1394         $usbbus = 'ehci';
1395     } else {
1396         $usbbus = 'uhci';
1397     }
1398
1399     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400 }
1401
1402 sub print_keyboarddevice_full {
1403     my ($conf, $arch, $machine) = @_;
1404
1405     return if $arch ne 'aarch64';
1406
1407     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408 }
1409
1410 my sub get_drive_id {
1411     my ($drive) = @_;
1412     return "$drive->{interface}$drive->{index}";
1413 }
1414
1415 sub print_drivedevice_full {
1416     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1417
1418     my $device = '';
1419     my $maxdev = 0;
1420
1421     my $drive_id = get_drive_id($drive);
1422     if ($drive->{interface} eq 'virtio') {
1423         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1424         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1425         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1426     } elsif ($drive->{interface} eq 'scsi') {
1427
1428         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1429         my $unit = $drive->{index} % $maxdev;
1430         my $devicetype = 'hd';
1431         my $path = '';
1432         if (drive_is_cdrom($drive)) {
1433             $devicetype = 'cd';
1434         } else {
1435             if ($drive->{file} =~ m|^/|) {
1436                 $path = $drive->{file};
1437                 if (my $info = path_is_scsi($path)) {
1438                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1439                         $devicetype = 'block';
1440                     } elsif ($info->{type} == 1) { # tape
1441                         $devicetype = 'generic';
1442                     }
1443                 }
1444             } else {
1445                  $path = PVE::Storage::path($storecfg, $drive->{file});
1446             }
1447
1448             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1449             my $version = extract_version($machine_type, kvm_user_version());
1450             if ($path =~ m/^iscsi\:\/\// &&
1451                !min_version($version, 4, 1)) {
1452                 $devicetype = 'generic';
1453             }
1454         }
1455
1456         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1458         } else {
1459             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1460                 .",lun=$drive->{index}";
1461         }
1462         $device .= ",drive=drive-$drive_id,id=$drive_id";
1463
1464         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1465             $device .= ",rotation_rate=1";
1466         }
1467         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1468
1469     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1470         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1471         my $controller = int($drive->{index} / $maxdev);
1472         my $unit = $drive->{index} % $maxdev;
1473         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1474
1475         $device = "ide-$devicetype";
1476         if ($drive->{interface} eq 'ide') {
1477             $device .= ",bus=ide.$controller,unit=$unit";
1478         } else {
1479             $device .= ",bus=ahci$controller.$unit";
1480         }
1481         $device .= ",drive=drive-$drive_id,id=$drive_id";
1482
1483         if ($devicetype eq 'hd') {
1484             if (my $model = $drive->{model}) {
1485                 $model = URI::Escape::uri_unescape($model);
1486                 $device .= ",model=$model";
1487             }
1488             if ($drive->{ssd}) {
1489                 $device .= ",rotation_rate=1";
1490             }
1491         }
1492         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1493     } elsif ($drive->{interface} eq 'usb') {
1494         die "implement me";
1495         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1496     } else {
1497         die "unsupported interface type";
1498     }
1499
1500     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1501
1502     if (my $serial = $drive->{serial}) {
1503         $serial = URI::Escape::uri_unescape($serial);
1504         $device .= ",serial=$serial";
1505     }
1506
1507
1508     return $device;
1509 }
1510
1511 sub get_initiator_name {
1512     my $initiator;
1513
1514     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1515     while (defined(my $line = <$fh>)) {
1516         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1517         $initiator = $1;
1518         last;
1519     }
1520     $fh->close();
1521
1522     return $initiator;
1523 }
1524
1525 sub print_drive_commandline_full {
1526     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1527
1528     my $path;
1529     my $volid = $drive->{file};
1530     my $format = $drive->{format};
1531     my $drive_id = get_drive_id($drive);
1532
1533     if (drive_is_cdrom($drive)) {
1534         $path = get_iso_path($storecfg, $vmid, $volid);
1535         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1536     } else {
1537         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1538         if ($storeid) {
1539             $path = PVE::Storage::path($storecfg, $volid);
1540             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1541             $format //= qemu_img_format($scfg, $volname);
1542         } else {
1543             $path = $volid;
1544             $format //= "raw";
1545         }
1546    }
1547
1548    my $is_rbd = $path =~ m/^rbd:/;
1549
1550     my $opts = '';
1551     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1552     foreach my $o (@qemu_drive_options) {
1553         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1554     }
1555
1556     # snapshot only accepts on|off
1557     if (defined($drive->{snapshot})) {
1558         my $v = $drive->{snapshot} ? 'on' : 'off';
1559         $opts .= ",snapshot=$v";
1560     }
1561
1562     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1563         my ($dir, $qmpname) = @$type;
1564         if (my $v = $drive->{"mbps$dir"}) {
1565             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1566         }
1567         if (my $v = $drive->{"mbps${dir}_max"}) {
1568             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1569         }
1570         if (my $v = $drive->{"bps${dir}_max_length"}) {
1571             $opts .= ",throttling.bps$qmpname-max-length=$v";
1572         }
1573         if (my $v = $drive->{"iops${dir}"}) {
1574             $opts .= ",throttling.iops$qmpname=$v";
1575         }
1576         if (my $v = $drive->{"iops${dir}_max"}) {
1577             $opts .= ",throttling.iops$qmpname-max=$v";
1578         }
1579         if (my $v = $drive->{"iops${dir}_max_length"}) {
1580             $opts .= ",throttling.iops$qmpname-max-length=$v";
1581         }
1582     }
1583
1584     if ($pbs_name) {
1585         $format = "rbd" if $is_rbd;
1586         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1587             if !$format;
1588         $opts .= ",format=alloc-track,file.driver=$format";
1589     } elsif ($format) {
1590         $opts .= ",format=$format";
1591     }
1592
1593     my $cache_direct = 0;
1594
1595     if (my $cache = $drive->{cache}) {
1596         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1597     } elsif (!drive_is_cdrom($drive)) {
1598         $opts .= ",cache=none";
1599         $cache_direct = 1;
1600     }
1601
1602     if (!$drive->{aio}) {
1603         if ($io_uring) {
1604             # io_uring supports all cache modes
1605             $opts .= ",aio=io_uring";
1606         } else {
1607             # aio native works only with O_DIRECT
1608             if($cache_direct) {
1609                 $opts .= ",aio=native";
1610             } else {
1611                 $opts .= ",aio=threads";
1612             }
1613         }
1614     }
1615
1616     if (!drive_is_cdrom($drive)) {
1617         my $detectzeroes;
1618         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1619             $detectzeroes = 'off';
1620         } elsif ($drive->{discard}) {
1621             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1622         } else {
1623             # This used to be our default with discard not being specified:
1624             $detectzeroes = 'on';
1625         }
1626
1627         # note: 'detect-zeroes' works per blockdev and we want it to persist
1628         # after the alloc-track is removed, so put it on 'file' directly
1629         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1630         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1631     }
1632
1633     if ($pbs_name) {
1634         $opts .= ",backing=$pbs_name";
1635         $opts .= ",auto-remove=on";
1636     }
1637
1638     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1639     my $file_param = "file";
1640     if ($pbs_name) {
1641         # non-rbd drivers require the underlying file to be a seperate block
1642         # node, so add a second .file indirection
1643         $file_param .= ".file" if !$is_rbd;
1644         $file_param .= ".filename";
1645     }
1646     my $pathinfo = $path ? "$file_param=$path," : '';
1647
1648     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1649 }
1650
1651 sub print_pbs_blockdev {
1652     my ($pbs_conf, $pbs_name) = @_;
1653     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1654     $blockdev .= ",repository=$pbs_conf->{repository}";
1655     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1656     $blockdev .= ",archive=$pbs_conf->{archive}";
1657     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1658     return $blockdev;
1659 }
1660
1661 sub print_netdevice_full {
1662     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1663
1664     my $device = $net->{model};
1665     if ($net->{model} eq 'virtio') {
1666          $device = 'virtio-net-pci';
1667      };
1668
1669     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1670     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1671     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1672         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1673         # and out of each queue plus one config interrupt and control vector queue
1674         my $vectors = $net->{queues} * 2 + 2;
1675         $tmpstr .= ",vectors=$vectors,mq=on";
1676     }
1677     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1678
1679     if (my $mtu = $net->{mtu}) {
1680         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1681             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1682             if ($mtu == 1) {
1683                  $mtu = $bridge_mtu;
1684             } elsif ($mtu < 576) {
1685                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1686             } elsif ($mtu > $bridge_mtu) {
1687                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1688             }
1689             $tmpstr .= ",host_mtu=$mtu";
1690         } else {
1691             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1692         }
1693     }
1694
1695     if ($use_old_bios_files) {
1696         my $romfile;
1697         if ($device eq 'virtio-net-pci') {
1698             $romfile = 'pxe-virtio.rom';
1699         } elsif ($device eq 'e1000') {
1700             $romfile = 'pxe-e1000.rom';
1701         } elsif ($device eq 'ne2k') {
1702             $romfile = 'pxe-ne2k_pci.rom';
1703         } elsif ($device eq 'pcnet') {
1704             $romfile = 'pxe-pcnet.rom';
1705         } elsif ($device eq 'rtl8139') {
1706             $romfile = 'pxe-rtl8139.rom';
1707         }
1708         $tmpstr .= ",romfile=$romfile" if $romfile;
1709     }
1710
1711     return $tmpstr;
1712 }
1713
1714 sub print_netdev_full {
1715     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1716
1717     my $i = '';
1718     if ($netid =~ m/^net(\d+)$/) {
1719         $i = int($1);
1720     }
1721
1722     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1723
1724     my $ifname = "tap${vmid}i$i";
1725
1726     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1727     die "interface name '$ifname' is too long (max 15 character)\n"
1728         if length($ifname) >= 16;
1729
1730     my $vhostparam = '';
1731     if (is_native($arch)) {
1732         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1733     }
1734
1735     my $vmname = $conf->{name} || "vm$vmid";
1736
1737     my $netdev = "";
1738     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1739
1740     if ($net->{bridge}) {
1741         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1742             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1743     } else {
1744         $netdev = "type=user,id=$netid,hostname=$vmname";
1745     }
1746
1747     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1748
1749     return $netdev;
1750 }
1751
1752 my $vga_map = {
1753     'cirrus' => 'cirrus-vga',
1754     'std' => 'VGA',
1755     'vmware' => 'vmware-svga',
1756     'virtio' => 'virtio-vga',
1757 };
1758
1759 sub print_vga_device {
1760     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1761
1762     my $type = $vga_map->{$vga->{type}};
1763     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1764         $type = 'virtio-gpu';
1765     }
1766     my $vgamem_mb = $vga->{memory};
1767
1768     my $max_outputs = '';
1769     if ($qxlnum) {
1770         $type = $id ? 'qxl' : 'qxl-vga';
1771
1772         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1773             # set max outputs so linux can have up to 4 qxl displays with one device
1774             if (min_version($machine_version, 4, 1)) {
1775                 $max_outputs = ",max_outputs=4";
1776             }
1777         }
1778     }
1779
1780     die "no devicetype for $vga->{type}\n" if !$type;
1781
1782     my $memory = "";
1783     if ($vgamem_mb) {
1784         if ($vga->{type} eq 'virtio') {
1785             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1786             $memory = ",max_hostmem=$bytes";
1787         } elsif ($qxlnum) {
1788             # from https://www.spice-space.org/multiple-monitors.html
1789             $memory = ",vgamem_mb=$vga->{memory}";
1790             my $ram = $vgamem_mb * 4;
1791             my $vram = $vgamem_mb * 2;
1792             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1793         } else {
1794             $memory = ",vgamem_mb=$vga->{memory}";
1795         }
1796     } elsif ($qxlnum && $id) {
1797         $memory = ",ram_size=67108864,vram_size=33554432";
1798     }
1799
1800     my $edidoff = "";
1801     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1802         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1803     }
1804
1805     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1806     my $vgaid = "vga" . ($id // '');
1807     my $pciaddr;
1808
1809     if ($q35 && $vgaid eq 'vga') {
1810         # the first display uses pcie.0 bus on q35 machines
1811         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1812     } else {
1813         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1814     }
1815
1816     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1817 }
1818
1819 sub parse_number_sets {
1820     my ($set) = @_;
1821     my $res = [];
1822     foreach my $part (split(/;/, $set)) {
1823         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1824             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1825             push @$res, [ $1, $2 ];
1826         } else {
1827             die "invalid range: $part\n";
1828         }
1829     }
1830     return $res;
1831 }
1832
1833 sub parse_numa {
1834     my ($data) = @_;
1835
1836     my $res = parse_property_string($numa_fmt, $data);
1837     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1838     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1839     return $res;
1840 }
1841
1842 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1843 sub parse_net {
1844     my ($data) = @_;
1845
1846     my $res = eval { parse_property_string($net_fmt, $data) };
1847     if ($@) {
1848         warn $@;
1849         return;
1850     }
1851     if (!defined($res->{macaddr})) {
1852         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1853         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1854     }
1855     return $res;
1856 }
1857
1858 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1859 sub parse_ipconfig {
1860     my ($data) = @_;
1861
1862     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1863     if ($@) {
1864         warn $@;
1865         return;
1866     }
1867
1868     if ($res->{gw} && !$res->{ip}) {
1869         warn 'gateway specified without specifying an IP address';
1870         return;
1871     }
1872     if ($res->{gw6} && !$res->{ip6}) {
1873         warn 'IPv6 gateway specified without specifying an IPv6 address';
1874         return;
1875     }
1876     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1877         warn 'gateway specified together with DHCP';
1878         return;
1879     }
1880     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1881         # gw6 + auto/dhcp
1882         warn "IPv6 gateway specified together with $res->{ip6} address";
1883         return;
1884     }
1885
1886     if (!$res->{ip} && !$res->{ip6}) {
1887         return { ip => 'dhcp', ip6 => 'dhcp' };
1888     }
1889
1890     return $res;
1891 }
1892
1893 sub print_net {
1894     my $net = shift;
1895
1896     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1897 }
1898
1899 sub add_random_macs {
1900     my ($settings) = @_;
1901
1902     foreach my $opt (keys %$settings) {
1903         next if $opt !~ m/^net(\d+)$/;
1904         my $net = parse_net($settings->{$opt});
1905         next if !$net;
1906         $settings->{$opt} = print_net($net);
1907     }
1908 }
1909
1910 sub vm_is_volid_owner {
1911     my ($storecfg, $vmid, $volid) = @_;
1912
1913     if ($volid !~  m|^/|) {
1914         my ($path, $owner);
1915         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1916         if ($owner && ($owner == $vmid)) {
1917             return 1;
1918         }
1919     }
1920
1921     return;
1922 }
1923
1924 sub vmconfig_register_unused_drive {
1925     my ($storecfg, $vmid, $conf, $drive) = @_;
1926
1927     if (drive_is_cloudinit($drive)) {
1928         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1929         warn $@ if $@;
1930     } elsif (!drive_is_cdrom($drive)) {
1931         my $volid = $drive->{file};
1932         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1933             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1934         }
1935     }
1936 }
1937
1938 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1939 my $smbios1_fmt = {
1940     uuid => {
1941         type => 'string',
1942         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1943         format_description => 'UUID',
1944         description => "Set SMBIOS1 UUID.",
1945         optional => 1,
1946     },
1947     version => {
1948         type => 'string',
1949         pattern => '[A-Za-z0-9+\/]+={0,2}',
1950         format_description => 'Base64 encoded string',
1951         description => "Set SMBIOS1 version.",
1952         optional => 1,
1953     },
1954     serial => {
1955         type => 'string',
1956         pattern => '[A-Za-z0-9+\/]+={0,2}',
1957         format_description => 'Base64 encoded string',
1958         description => "Set SMBIOS1 serial number.",
1959         optional => 1,
1960     },
1961     manufacturer => {
1962         type => 'string',
1963         pattern => '[A-Za-z0-9+\/]+={0,2}',
1964         format_description => 'Base64 encoded string',
1965         description => "Set SMBIOS1 manufacturer.",
1966         optional => 1,
1967     },
1968     product => {
1969         type => 'string',
1970         pattern => '[A-Za-z0-9+\/]+={0,2}',
1971         format_description => 'Base64 encoded string',
1972         description => "Set SMBIOS1 product ID.",
1973         optional => 1,
1974     },
1975     sku => {
1976         type => 'string',
1977         pattern => '[A-Za-z0-9+\/]+={0,2}',
1978         format_description => 'Base64 encoded string',
1979         description => "Set SMBIOS1 SKU string.",
1980         optional => 1,
1981     },
1982     family => {
1983         type => 'string',
1984         pattern => '[A-Za-z0-9+\/]+={0,2}',
1985         format_description => 'Base64 encoded string',
1986         description => "Set SMBIOS1 family string.",
1987         optional => 1,
1988     },
1989     base64 => {
1990         type => 'boolean',
1991         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1992         optional => 1,
1993     },
1994 };
1995
1996 sub parse_smbios1 {
1997     my ($data) = @_;
1998
1999     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2000     warn $@ if $@;
2001     return $res;
2002 }
2003
2004 sub print_smbios1 {
2005     my ($smbios1) = @_;
2006     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2010
2011 sub parse_watchdog {
2012     my ($value) = @_;
2013
2014     return if !$value;
2015
2016     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2017     warn $@ if $@;
2018     return $res;
2019 }
2020
2021 sub parse_guest_agent {
2022     my ($conf) = @_;
2023
2024     return {} if !defined($conf->{agent});
2025
2026     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2027     warn $@ if $@;
2028
2029     # if the agent is disabled ignore the other potentially set properties
2030     return {} if !$res->{enabled};
2031     return $res;
2032 }
2033
2034 sub get_qga_key {
2035     my ($conf, $key) = @_;
2036     return undef if !defined($conf->{agent});
2037
2038     my $agent = parse_guest_agent($conf);
2039     return $agent->{$key};
2040 }
2041
2042 sub parse_vga {
2043     my ($value) = @_;
2044
2045     return {} if !$value;
2046     my $res = eval { parse_property_string($vga_fmt, $value) };
2047     warn $@ if $@;
2048     return $res;
2049 }
2050
2051 sub parse_rng {
2052     my ($value) = @_;
2053
2054     return if !$value;
2055
2056     my $res = eval { parse_property_string($rng_fmt, $value) };
2057     warn $@ if $@;
2058     return $res;
2059 }
2060
2061 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2062 sub verify_usb_device {
2063     my ($value, $noerr) = @_;
2064
2065     return $value if parse_usb_device($value);
2066
2067     return if $noerr;
2068
2069     die "unable to parse usb device\n";
2070 }
2071
2072 # add JSON properties for create and set function
2073 sub json_config_properties {
2074     my $prop = shift;
2075
2076     foreach my $opt (keys %$confdesc) {
2077         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2078             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2079         $prop->{$opt} = $confdesc->{$opt};
2080     }
2081
2082     return $prop;
2083 }
2084
2085 # return copy of $confdesc_cloudinit to generate documentation
2086 sub cloudinit_config_properties {
2087
2088     return dclone($confdesc_cloudinit);
2089 }
2090
2091 sub check_type {
2092     my ($key, $value) = @_;
2093
2094     die "unknown setting '$key'\n" if !$confdesc->{$key};
2095
2096     my $type = $confdesc->{$key}->{type};
2097
2098     if (!defined($value)) {
2099         die "got undefined value\n";
2100     }
2101
2102     if ($value =~ m/[\n\r]/) {
2103         die "property contains a line feed\n";
2104     }
2105
2106     if ($type eq 'boolean') {
2107         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2108         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2109         die "type check ('boolean') failed - got '$value'\n";
2110     } elsif ($type eq 'integer') {
2111         return int($1) if $value =~ m/^(\d+)$/;
2112         die "type check ('integer') failed - got '$value'\n";
2113     } elsif ($type eq 'number') {
2114         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2115         die "type check ('number') failed - got '$value'\n";
2116     } elsif ($type eq 'string') {
2117         if (my $fmt = $confdesc->{$key}->{format}) {
2118             PVE::JSONSchema::check_format($fmt, $value);
2119             return $value;
2120         }
2121         $value =~ s/^\"(.*)\"$/$1/;
2122         return $value;
2123     } else {
2124         die "internal error"
2125     }
2126 }
2127
2128 sub destroy_vm {
2129     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2130
2131     my $conf = PVE::QemuConfig->load_config($vmid);
2132
2133     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2134
2135     if ($conf->{template}) {
2136         # check if any base image is still used by a linked clone
2137         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2138                 my ($ds, $drive) = @_;
2139                 return if drive_is_cdrom($drive);
2140
2141                 my $volid = $drive->{file};
2142                 return if !$volid || $volid =~ m|^/|;
2143
2144                 die "base volume '$volid' is still in use by linked cloned\n"
2145                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2146
2147         });
2148     }
2149
2150     my $remove_owned_drive = sub {
2151         my ($ds, $drive) = @_;
2152         return if drive_is_cdrom($drive, 1);
2153
2154         my $volid = $drive->{file};
2155         return if !$volid || $volid =~ m|^/|;
2156
2157         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2158         return if !$path || !$owner || ($owner != $vmid);
2159
2160         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2161         warn "Could not remove disk '$volid', check manually: $@" if $@;
2162     };
2163
2164     # only remove disks owned by this VM (referenced in the config)
2165     my $include_opts = {
2166         include_unused => 1,
2167         extra_keys => ['vmstate'],
2168     };
2169     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2170
2171     for my $snap (values %{$conf->{snapshots}}) {
2172         next if !defined($snap->{vmstate});
2173         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2174         next if !defined($drive);
2175         $remove_owned_drive->('vmstate', $drive);
2176     }
2177
2178     if ($purge_unreferenced) { # also remove unreferenced disk
2179         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2180         PVE::Storage::foreach_volid($vmdisks, sub {
2181             my ($volid, $sid, $volname, $d) = @_;
2182             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2183             warn $@ if $@;
2184         });
2185     }
2186
2187     if (defined $replacement_conf) {
2188         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2189     } else {
2190         PVE::QemuConfig->destroy_config($vmid);
2191     }
2192 }
2193
2194 sub parse_vm_config {
2195     my ($filename, $raw) = @_;
2196
2197     return if !defined($raw);
2198
2199     my $res = {
2200         digest => Digest::SHA::sha1_hex($raw),
2201         snapshots => {},
2202         pending => {},
2203     };
2204
2205     $filename =~ m|/qemu-server/(\d+)\.conf$|
2206         || die "got strange filename '$filename'";
2207
2208     my $vmid = $1;
2209
2210     my $conf = $res;
2211     my $descr;
2212     my $section = '';
2213
2214     my @lines = split(/\n/, $raw);
2215     foreach my $line (@lines) {
2216         next if $line =~ m/^\s*$/;
2217
2218         if ($line =~ m/^\[PENDING\]\s*$/i) {
2219             $section = 'pending';
2220             if (defined($descr)) {
2221                 $descr =~ s/\s+$//;
2222                 $conf->{description} = $descr;
2223             }
2224             $descr = undef;
2225             $conf = $res->{$section} = {};
2226             next;
2227
2228         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2229             $section = $1;
2230             if (defined($descr)) {
2231                 $descr =~ s/\s+$//;
2232                 $conf->{description} = $descr;
2233             }
2234             $descr = undef;
2235             $conf = $res->{snapshots}->{$section} = {};
2236             next;
2237         }
2238
2239         if ($line =~ m/^\#(.*)\s*$/) {
2240             $descr = '' if !defined($descr);
2241             $descr .= PVE::Tools::decode_text($1) . "\n";
2242             next;
2243         }
2244
2245         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2246             $descr = '' if !defined($descr);
2247             $descr .= PVE::Tools::decode_text($2);
2248         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2249             $conf->{snapstate} = $1;
2250         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2251             my $key = $1;
2252             my $value = $2;
2253             $conf->{$key} = $value;
2254         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2255             my $value = $1;
2256             if ($section eq 'pending') {
2257                 $conf->{delete} = $value; # we parse this later
2258             } else {
2259                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2260             }
2261         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2262             my $key = $1;
2263             my $value = $2;
2264             eval { $value = check_type($key, $value); };
2265             if ($@) {
2266                 warn "vm $vmid - unable to parse value of '$key' - $@";
2267             } else {
2268                 $key = 'ide2' if $key eq 'cdrom';
2269                 my $fmt = $confdesc->{$key}->{format};
2270                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2271                     my $v = parse_drive($key, $value);
2272                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2273                         $v->{file} = $volid;
2274                         $value = print_drive($v);
2275                     } else {
2276                         warn "vm $vmid - unable to parse value of '$key'\n";
2277                         next;
2278                     }
2279                 }
2280
2281                 $conf->{$key} = $value;
2282             }
2283         } else {
2284             warn "vm $vmid - unable to parse config: $line\n";
2285         }
2286     }
2287
2288     if (defined($descr)) {
2289         $descr =~ s/\s+$//;
2290         $conf->{description} = $descr;
2291     }
2292     delete $res->{snapstate}; # just to be sure
2293
2294     return $res;
2295 }
2296
2297 sub write_vm_config {
2298     my ($filename, $conf) = @_;
2299
2300     delete $conf->{snapstate}; # just to be sure
2301
2302     if ($conf->{cdrom}) {
2303         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2304         $conf->{ide2} = $conf->{cdrom};
2305         delete $conf->{cdrom};
2306     }
2307
2308     # we do not use 'smp' any longer
2309     if ($conf->{sockets}) {
2310         delete $conf->{smp};
2311     } elsif ($conf->{smp}) {
2312         $conf->{sockets} = $conf->{smp};
2313         delete $conf->{cores};
2314         delete $conf->{smp};
2315     }
2316
2317     my $used_volids = {};
2318
2319     my $cleanup_config = sub {
2320         my ($cref, $pending, $snapname) = @_;
2321
2322         foreach my $key (keys %$cref) {
2323             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2324                 $key eq 'snapstate' || $key eq 'pending';
2325             my $value = $cref->{$key};
2326             if ($key eq 'delete') {
2327                 die "propertry 'delete' is only allowed in [PENDING]\n"
2328                     if !$pending;
2329                 # fixme: check syntax?
2330                 next;
2331             }
2332             eval { $value = check_type($key, $value); };
2333             die "unable to parse value of '$key' - $@" if $@;
2334
2335             $cref->{$key} = $value;
2336
2337             if (!$snapname && is_valid_drivename($key)) {
2338                 my $drive = parse_drive($key, $value);
2339                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2340             }
2341         }
2342     };
2343
2344     &$cleanup_config($conf);
2345
2346     &$cleanup_config($conf->{pending}, 1);
2347
2348     foreach my $snapname (keys %{$conf->{snapshots}}) {
2349         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2350         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2351     }
2352
2353     # remove 'unusedX' settings if we re-add a volume
2354     foreach my $key (keys %$conf) {
2355         my $value = $conf->{$key};
2356         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2357             delete $conf->{$key};
2358         }
2359     }
2360
2361     my $generate_raw_config = sub {
2362         my ($conf, $pending) = @_;
2363
2364         my $raw = '';
2365
2366         # add description as comment to top of file
2367         if (defined(my $descr = $conf->{description})) {
2368             if ($descr) {
2369                 foreach my $cl (split(/\n/, $descr)) {
2370                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2371                 }
2372             } else {
2373                 $raw .= "#\n" if $pending;
2374             }
2375         }
2376
2377         foreach my $key (sort keys %$conf) {
2378             next if $key =~ /^(digest|description|pending|snapshots)$/;
2379             $raw .= "$key: $conf->{$key}\n";
2380         }
2381         return $raw;
2382     };
2383
2384     my $raw = &$generate_raw_config($conf);
2385
2386     if (scalar(keys %{$conf->{pending}})){
2387         $raw .= "\n[PENDING]\n";
2388         $raw .= &$generate_raw_config($conf->{pending}, 1);
2389     }
2390
2391     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2392         $raw .= "\n[$snapname]\n";
2393         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2394     }
2395
2396     return $raw;
2397 }
2398
2399 sub load_defaults {
2400
2401     my $res = {};
2402
2403     # we use static defaults from our JSON schema configuration
2404     foreach my $key (keys %$confdesc) {
2405         if (defined(my $default = $confdesc->{$key}->{default})) {
2406             $res->{$key} = $default;
2407         }
2408     }
2409
2410     return $res;
2411 }
2412
2413 sub config_list {
2414     my $vmlist = PVE::Cluster::get_vmlist();
2415     my $res = {};
2416     return $res if !$vmlist || !$vmlist->{ids};
2417     my $ids = $vmlist->{ids};
2418     my $nodename = nodename();
2419
2420     foreach my $vmid (keys %$ids) {
2421         my $d = $ids->{$vmid};
2422         next if !$d->{node} || $d->{node} ne $nodename;
2423         next if !$d->{type} || $d->{type} ne 'qemu';
2424         $res->{$vmid}->{exists} = 1;
2425     }
2426     return $res;
2427 }
2428
2429 # test if VM uses local resources (to prevent migration)
2430 sub check_local_resources {
2431     my ($conf, $noerr) = @_;
2432
2433     my @loc_res = ();
2434
2435     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2436     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2437
2438     push @loc_res, "ivshmem" if $conf->{ivshmem};
2439
2440     foreach my $k (keys %$conf) {
2441         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2442         # sockets are safe: they will recreated be on the target side post-migrate
2443         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2444         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2445     }
2446
2447     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2448
2449     return \@loc_res;
2450 }
2451
2452 # check if used storages are available on all nodes (use by migrate)
2453 sub check_storage_availability {
2454     my ($storecfg, $conf, $node) = @_;
2455
2456     PVE::QemuConfig->foreach_volume($conf, sub {
2457         my ($ds, $drive) = @_;
2458
2459         my $volid = $drive->{file};
2460         return if !$volid;
2461
2462         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2463         return if !$sid;
2464
2465         # check if storage is available on both nodes
2466         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2467         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2468
2469         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2470
2471         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2472             if !$scfg->{content}->{$vtype};
2473    });
2474 }
2475
2476 # list nodes where all VM images are available (used by has_feature API)
2477 sub shared_nodes {
2478     my ($conf, $storecfg) = @_;
2479
2480     my $nodelist = PVE::Cluster::get_nodelist();
2481     my $nodehash = { map { $_ => 1 } @$nodelist };
2482     my $nodename = nodename();
2483
2484     PVE::QemuConfig->foreach_volume($conf, sub {
2485         my ($ds, $drive) = @_;
2486
2487         my $volid = $drive->{file};
2488         return if !$volid;
2489
2490         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2491         if ($storeid) {
2492             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2493             if ($scfg->{disable}) {
2494                 $nodehash = {};
2495             } elsif (my $avail = $scfg->{nodes}) {
2496                 foreach my $node (keys %$nodehash) {
2497                     delete $nodehash->{$node} if !$avail->{$node};
2498                 }
2499             } elsif (!$scfg->{shared}) {
2500                 foreach my $node (keys %$nodehash) {
2501                     delete $nodehash->{$node} if $node ne $nodename
2502                 }
2503             }
2504         }
2505     });
2506
2507     return $nodehash
2508 }
2509
2510 sub check_local_storage_availability {
2511     my ($conf, $storecfg) = @_;
2512
2513     my $nodelist = PVE::Cluster::get_nodelist();
2514     my $nodehash = { map { $_ => {} } @$nodelist };
2515
2516     PVE::QemuConfig->foreach_volume($conf, sub {
2517         my ($ds, $drive) = @_;
2518
2519         my $volid = $drive->{file};
2520         return if !$volid;
2521
2522         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2523         if ($storeid) {
2524             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2525
2526             if ($scfg->{disable}) {
2527                 foreach my $node (keys %$nodehash) {
2528                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2529                 }
2530             } elsif (my $avail = $scfg->{nodes}) {
2531                 foreach my $node (keys %$nodehash) {
2532                     if (!$avail->{$node}) {
2533                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2534                     }
2535                 }
2536             }
2537         }
2538     });
2539
2540     foreach my $node (values %$nodehash) {
2541         if (my $unavail = $node->{unavailable_storages}) {
2542             $node->{unavailable_storages} = [ sort keys %$unavail ];
2543         }
2544     }
2545
2546     return $nodehash
2547 }
2548
2549 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2550 sub check_running {
2551     my ($vmid, $nocheck, $node) = @_;
2552
2553     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2554     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2555 }
2556
2557 sub vzlist {
2558
2559     my $vzlist = config_list();
2560
2561     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2562
2563     while (defined(my $de = $fd->read)) {
2564         next if $de !~ m/^(\d+)\.pid$/;
2565         my $vmid = $1;
2566         next if !defined($vzlist->{$vmid});
2567         if (my $pid = check_running($vmid)) {
2568             $vzlist->{$vmid}->{pid} = $pid;
2569         }
2570     }
2571
2572     return $vzlist;
2573 }
2574
2575 our $vmstatus_return_properties = {
2576     vmid => get_standard_option('pve-vmid'),
2577     status => {
2578         description => "Qemu process status.",
2579         type => 'string',
2580         enum => ['stopped', 'running'],
2581     },
2582     maxmem => {
2583         description => "Maximum memory in bytes.",
2584         type => 'integer',
2585         optional => 1,
2586         renderer => 'bytes',
2587     },
2588     maxdisk => {
2589         description => "Root disk size in bytes.",
2590         type => 'integer',
2591         optional => 1,
2592         renderer => 'bytes',
2593     },
2594     name => {
2595         description => "VM name.",
2596         type => 'string',
2597         optional => 1,
2598     },
2599     qmpstatus => {
2600         description => "Qemu QMP agent status.",
2601         type => 'string',
2602         optional => 1,
2603     },
2604     pid => {
2605         description => "PID of running qemu process.",
2606         type => 'integer',
2607         optional => 1,
2608     },
2609     uptime => {
2610         description => "Uptime.",
2611         type => 'integer',
2612         optional => 1,
2613         renderer => 'duration',
2614     },
2615     cpus => {
2616         description => "Maximum usable CPUs.",
2617         type => 'number',
2618         optional => 1,
2619     },
2620     lock => {
2621         description => "The current config lock, if any.",
2622         type => 'string',
2623         optional => 1,
2624     },
2625     tags => {
2626         description  => "The current configured tags, if any",
2627         type => 'string',
2628         optional => 1,
2629     },
2630     'running-machine' => {
2631         description => "The currently running machine type (if running).",
2632         type => 'string',
2633         optional => 1,
2634     },
2635     'running-qemu' => {
2636         description => "The currently running QEMU version (if running).",
2637         type => 'string',
2638         optional => 1,
2639     },
2640 };
2641
2642 my $last_proc_pid_stat;
2643
2644 # get VM status information
2645 # This must be fast and should not block ($full == false)
2646 # We only query KVM using QMP if $full == true (this can be slow)
2647 sub vmstatus {
2648     my ($opt_vmid, $full) = @_;
2649
2650     my $res = {};
2651
2652     my $storecfg = PVE::Storage::config();
2653
2654     my $list = vzlist();
2655     my $defaults = load_defaults();
2656
2657     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2658
2659     my $cpucount = $cpuinfo->{cpus} || 1;
2660
2661     foreach my $vmid (keys %$list) {
2662         next if $opt_vmid && ($vmid ne $opt_vmid);
2663
2664         my $conf = PVE::QemuConfig->load_config($vmid);
2665
2666         my $d = { vmid => int($vmid) };
2667         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2668
2669         # fixme: better status?
2670         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2671
2672         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2673         if (defined($size)) {
2674             $d->{disk} = 0; # no info available
2675             $d->{maxdisk} = $size;
2676         } else {
2677             $d->{disk} = 0;
2678             $d->{maxdisk} = 0;
2679         }
2680
2681         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2682             * ($conf->{cores} || $defaults->{cores});
2683         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2684         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2685
2686         $d->{name} = $conf->{name} || "VM $vmid";
2687         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2688             : $defaults->{memory}*(1024*1024);
2689
2690         if ($conf->{balloon}) {
2691             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2692             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2693                 : $defaults->{shares};
2694         }
2695
2696         $d->{uptime} = 0;
2697         $d->{cpu} = 0;
2698         $d->{mem} = 0;
2699
2700         $d->{netout} = 0;
2701         $d->{netin} = 0;
2702
2703         $d->{diskread} = 0;
2704         $d->{diskwrite} = 0;
2705
2706         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2707
2708         $d->{serial} = 1 if conf_has_serial($conf);
2709         $d->{lock} = $conf->{lock} if $conf->{lock};
2710         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2711
2712         $res->{$vmid} = $d;
2713     }
2714
2715     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2716     foreach my $dev (keys %$netdev) {
2717         next if $dev !~ m/^tap([1-9]\d*)i/;
2718         my $vmid = $1;
2719         my $d = $res->{$vmid};
2720         next if !$d;
2721
2722         $d->{netout} += $netdev->{$dev}->{receive};
2723         $d->{netin} += $netdev->{$dev}->{transmit};
2724
2725         if ($full) {
2726             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2727             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2728         }
2729
2730     }
2731
2732     my $ctime = gettimeofday;
2733
2734     foreach my $vmid (keys %$list) {
2735
2736         my $d = $res->{$vmid};
2737         my $pid = $d->{pid};
2738         next if !$pid;
2739
2740         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2741         next if !$pstat; # not running
2742
2743         my $used = $pstat->{utime} + $pstat->{stime};
2744
2745         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2746
2747         if ($pstat->{vsize}) {
2748             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2749         }
2750
2751         my $old = $last_proc_pid_stat->{$pid};
2752         if (!$old) {
2753             $last_proc_pid_stat->{$pid} = {
2754                 time => $ctime,
2755                 used => $used,
2756                 cpu => 0,
2757             };
2758             next;
2759         }
2760
2761         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2762
2763         if ($dtime > 1000) {
2764             my $dutime = $used -  $old->{used};
2765
2766             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2767             $last_proc_pid_stat->{$pid} = {
2768                 time => $ctime,
2769                 used => $used,
2770                 cpu => $d->{cpu},
2771             };
2772         } else {
2773             $d->{cpu} = $old->{cpu};
2774         }
2775     }
2776
2777     return $res if !$full;
2778
2779     my $qmpclient = PVE::QMPClient->new();
2780
2781     my $ballooncb = sub {
2782         my ($vmid, $resp) = @_;
2783
2784         my $info = $resp->{'return'};
2785         return if !$info->{max_mem};
2786
2787         my $d = $res->{$vmid};
2788
2789         # use memory assigned to VM
2790         $d->{maxmem} = $info->{max_mem};
2791         $d->{balloon} = $info->{actual};
2792
2793         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2794             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2795             $d->{freemem} = $info->{free_mem};
2796         }
2797
2798         $d->{ballooninfo} = $info;
2799     };
2800
2801     my $blockstatscb = sub {
2802         my ($vmid, $resp) = @_;
2803         my $data = $resp->{'return'} || [];
2804         my $totalrdbytes = 0;
2805         my $totalwrbytes = 0;
2806
2807         for my $blockstat (@$data) {
2808             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2809             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2810
2811             $blockstat->{device} =~ s/drive-//;
2812             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2813         }
2814         $res->{$vmid}->{diskread} = $totalrdbytes;
2815         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2816     };
2817
2818     my $machinecb = sub {
2819         my ($vmid, $resp) = @_;
2820         my $data = $resp->{'return'} || [];
2821
2822         $res->{$vmid}->{'running-machine'} =
2823             PVE::QemuServer::Machine::current_from_query_machines($data);
2824     };
2825
2826     my $versioncb = sub {
2827         my ($vmid, $resp) = @_;
2828         my $data = $resp->{'return'} // {};
2829         my $version = 'unknown';
2830
2831         if (my $v = $data->{qemu}) {
2832             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2833         }
2834
2835         $res->{$vmid}->{'running-qemu'} = $version;
2836     };
2837
2838     my $statuscb = sub {
2839         my ($vmid, $resp) = @_;
2840
2841         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2842         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2843         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2844         # this fails if ballon driver is not loaded, so this must be
2845         # the last commnand (following command are aborted if this fails).
2846         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2847
2848         my $status = 'unknown';
2849         if (!defined($status = $resp->{'return'}->{status})) {
2850             warn "unable to get VM status\n";
2851             return;
2852         }
2853
2854         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2855     };
2856
2857     foreach my $vmid (keys %$list) {
2858         next if $opt_vmid && ($vmid ne $opt_vmid);
2859         next if !$res->{$vmid}->{pid}; # not running
2860         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2861     }
2862
2863     $qmpclient->queue_execute(undef, 2);
2864
2865     foreach my $vmid (keys %$list) {
2866         next if $opt_vmid && ($vmid ne $opt_vmid);
2867         next if !$res->{$vmid}->{pid}; #not running
2868
2869         # we can't use the $qmpclient since it might have already aborted on
2870         # 'query-balloon', but this might also fail for older versions...
2871         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2872         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2873     }
2874
2875     foreach my $vmid (keys %$list) {
2876         next if $opt_vmid && ($vmid ne $opt_vmid);
2877         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2878     }
2879
2880     return $res;
2881 }
2882
2883 sub conf_has_serial {
2884     my ($conf) = @_;
2885
2886     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2887         if ($conf->{"serial$i"}) {
2888             return 1;
2889         }
2890     }
2891
2892     return 0;
2893 }
2894
2895 sub conf_has_audio {
2896     my ($conf, $id) = @_;
2897
2898     $id //= 0;
2899     my $audio = $conf->{"audio$id"};
2900     return if !defined($audio);
2901
2902     my $audioproperties = parse_property_string($audio_fmt, $audio);
2903     my $audiodriver = $audioproperties->{driver} // 'spice';
2904
2905     return {
2906         dev => $audioproperties->{device},
2907         dev_id => "audiodev$id",
2908         backend => $audiodriver,
2909         backend_id => "$audiodriver-backend${id}",
2910     };
2911 }
2912
2913 sub audio_devs {
2914     my ($audio, $audiopciaddr, $machine_version) = @_;
2915
2916     my $devs = [];
2917
2918     my $id = $audio->{dev_id};
2919     my $audiodev = "";
2920     if (min_version($machine_version, 4, 2)) {
2921         $audiodev = ",audiodev=$audio->{backend_id}";
2922     }
2923
2924     if ($audio->{dev} eq 'AC97') {
2925         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2926     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2927         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2928         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2929         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2930     } else {
2931         die "unkown audio device '$audio->{dev}', implement me!";
2932     }
2933
2934     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2935
2936     return $devs;
2937 }
2938
2939 sub vga_conf_has_spice {
2940     my ($vga) = @_;
2941
2942     my $vgaconf = parse_vga($vga);
2943     my $vgatype = $vgaconf->{type};
2944     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2945
2946     return $1 || 1;
2947 }
2948
2949 sub is_native($) {
2950     my ($arch) = @_;
2951     return get_host_arch() eq $arch;
2952 }
2953
2954 sub get_vm_arch {
2955     my ($conf) = @_;
2956     return $conf->{arch} // get_host_arch();
2957 }
2958
2959 my $default_machines = {
2960     x86_64 => 'pc',
2961     aarch64 => 'virt',
2962 };
2963
2964 sub get_installed_machine_version {
2965     my ($kvmversion) = @_;
2966     $kvmversion = kvm_user_version() if !defined($kvmversion);
2967     $kvmversion =~ m/^(\d+\.\d+)/;
2968     return $1;
2969 }
2970
2971 sub windows_get_pinned_machine_version {
2972     my ($machine, $base_version, $kvmversion) = @_;
2973
2974     my $pin_version = $base_version;
2975     if (!defined($base_version) ||
2976         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2977     ) {
2978         $pin_version = get_installed_machine_version($kvmversion);
2979     }
2980     if (!$machine || $machine eq 'pc') {
2981         $machine = "pc-i440fx-$pin_version";
2982     } elsif ($machine eq 'q35') {
2983         $machine = "pc-q35-$pin_version";
2984     } elsif ($machine eq 'virt') {
2985         $machine = "virt-$pin_version";
2986     } else {
2987         warn "unknown machine type '$machine', not touching that!\n";
2988     }
2989
2990     return $machine;
2991 }
2992
2993 sub get_vm_machine {
2994     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2995
2996     my $machine = $forcemachine || $conf->{machine};
2997
2998     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2999         $kvmversion //= kvm_user_version();
3000         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3001         # layout which confuses windows quite a bit and may result in various regressions..
3002         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3003         if (windows_version($conf->{ostype})) {
3004             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3005         }
3006         $arch //= 'x86_64';
3007         $machine ||= $default_machines->{$arch};
3008         if ($add_pve_version) {
3009             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3010             $machine .= "+pve$pvever";
3011         }
3012     }
3013
3014     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3015         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3016         $machine = $1 if $is_pxe;
3017
3018         # for version-pinned machines that do not include a pve-version (e.g.
3019         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3020         $machine .= '+pve0';
3021
3022         $machine .= '.pxe' if $is_pxe;
3023     }
3024
3025     return $machine;
3026 }
3027
3028 sub get_ovmf_files($) {
3029     my ($arch) = @_;
3030
3031     my $ovmf = $OVMF->{$arch}
3032         or die "no OVMF images known for architecture '$arch'\n";
3033
3034     return @$ovmf;
3035 }
3036
3037 my $Arch2Qemu = {
3038     aarch64 => '/usr/bin/qemu-system-aarch64',
3039     x86_64 => '/usr/bin/qemu-system-x86_64',
3040 };
3041 sub get_command_for_arch($) {
3042     my ($arch) = @_;
3043     return '/usr/bin/kvm' if is_native($arch);
3044
3045     my $cmd = $Arch2Qemu->{$arch}
3046         or die "don't know how to emulate architecture '$arch'\n";
3047     return $cmd;
3048 }
3049
3050 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3051 # to use in a QEMU command line (-cpu element), first array_intersect the result
3052 # of query_supported_ with query_understood_. This is necessary because:
3053 #
3054 # a) query_understood_ returns flags the host cannot use and
3055 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3056 #    flags, but CPU settings - with most of them being flags. Those settings
3057 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3058 #
3059 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3060 # expensive. If you need the value returned from this, you can get it much
3061 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3062 # $accel being 'kvm' or 'tcg'.
3063 #
3064 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3065 # changes, automatically populating pmxcfs.
3066 #
3067 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3068 # since kvm and tcg machines support different flags
3069 #
3070 sub query_supported_cpu_flags {
3071     my ($arch) = @_;
3072
3073     $arch //= get_host_arch();
3074     my $default_machine = $default_machines->{$arch};
3075
3076     my $flags = {};
3077
3078     # FIXME: Once this is merged, the code below should work for ARM as well:
3079     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3080     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3081         $arch eq "aarch64";
3082
3083     my $kvm_supported = defined(kvm_version());
3084     my $qemu_cmd = get_command_for_arch($arch);
3085     my $fakevmid = -1;
3086     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3087
3088     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3089     my $query_supported_run_qemu = sub {
3090         my ($kvm) = @_;
3091
3092         my $flags = {};
3093         my $cmd = [
3094             $qemu_cmd,
3095             '-machine', $default_machine,
3096             '-display', 'none',
3097             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3098             '-mon', 'chardev=qmp,mode=control',
3099             '-pidfile', $pidfile,
3100             '-S', '-daemonize'
3101         ];
3102
3103         if (!$kvm) {
3104             push @$cmd, '-accel', 'tcg';
3105         }
3106
3107         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3108         die "QEMU flag querying VM exited with code " . $rc if $rc;
3109
3110         eval {
3111             my $cmd_result = mon_cmd(
3112                 $fakevmid,
3113                 'query-cpu-model-expansion',
3114                 type => 'full',
3115                 model => { name => 'host' }
3116             );
3117
3118             my $props = $cmd_result->{model}->{props};
3119             foreach my $prop (keys %$props) {
3120                 next if $props->{$prop} ne '1';
3121                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3122                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3123                 # We only keep those with underscores, to match /proc/cpuinfo
3124                 $prop =~ s/\.|-/_/g;
3125                 $flags->{$prop} = 1;
3126             }
3127         };
3128         my $err = $@;
3129
3130         # force stop with 10 sec timeout and 'nocheck'
3131         # always stop, even if QMP failed
3132         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3133
3134         die $err if $err;
3135
3136         return [ sort keys %$flags ];
3137     };
3138
3139     # We need to query QEMU twice, since KVM and TCG have different supported flags
3140     PVE::QemuConfig->lock_config($fakevmid, sub {
3141         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3142         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3143
3144         if ($kvm_supported) {
3145             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3146             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3147         }
3148     });
3149
3150     return $flags;
3151 }
3152
3153 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3154 my $understood_cpu_flag_dir = "/usr/share/kvm";
3155 sub query_understood_cpu_flags {
3156     my $arch = get_host_arch();
3157     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3158
3159     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3160         if ! -e $filepath;
3161
3162     my $raw = file_get_contents($filepath);
3163     $raw =~ s/^\s+|\s+$//g;
3164     my @flags = split(/\s+/, $raw);
3165
3166     return \@flags;
3167 }
3168
3169 sub config_to_command {
3170     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3171         $pbs_backing) = @_;
3172
3173     my $cmd = [];
3174     my $globalFlags = [];
3175     my $machineFlags = [];
3176     my $rtcFlags = [];
3177     my $devices = [];
3178     my $pciaddr = '';
3179     my $bridges = {};
3180     my $ostype = $conf->{ostype};
3181     my $winversion = windows_version($ostype);
3182     my $kvm = $conf->{kvm};
3183     my $nodename = nodename();
3184
3185     my $arch = get_vm_arch($conf);
3186     my $kvm_binary = get_command_for_arch($arch);
3187     my $kvmver = kvm_user_version($kvm_binary);
3188
3189     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3190         $kvmver //= "undefined";
3191         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3192     }
3193
3194     my $add_pve_version = min_version($kvmver, 4, 1);
3195
3196     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3197     my $machine_version = extract_version($machine_type, $kvmver);
3198     $kvm //= 1 if is_native($arch);
3199
3200     $machine_version =~ m/(\d+)\.(\d+)/;
3201     my ($machine_major, $machine_minor) = ($1, $2);
3202
3203     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3204         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3205     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3206         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3207             ." please upgrade node '$nodename'\n"
3208     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3209         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3210         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3211             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3212             ." node '$nodename'\n";
3213     }
3214
3215     # if a specific +pve version is required for a feature, use $version_guard
3216     # instead of min_version to allow machines to be run with the minimum
3217     # required version
3218     my $required_pve_version = 0;
3219     my $version_guard = sub {
3220         my ($major, $minor, $pve) = @_;
3221         return 0 if !min_version($machine_version, $major, $minor, $pve);
3222         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3223         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3224         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3225         return 1;
3226     };
3227
3228     if ($kvm && !defined kvm_version()) {
3229         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3230             ." or enable in BIOS.\n";
3231     }
3232
3233     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3234     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3235     my $use_old_bios_files = undef;
3236     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3237
3238     my $cpuunits = defined($conf->{cpuunits}) ?
3239             $conf->{cpuunits} : $defaults->{cpuunits};
3240
3241     push @$cmd, $kvm_binary;
3242
3243     push @$cmd, '-id', $vmid;
3244
3245     my $vmname = $conf->{name} || "vm$vmid";
3246
3247     push @$cmd, '-name', $vmname;
3248
3249     push @$cmd, '-no-shutdown';
3250
3251     my $use_virtio = 0;
3252
3253     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3254     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3255     push @$cmd, '-mon', "chardev=qmp,mode=control";
3256
3257     if (min_version($machine_version, 2, 12)) {
3258         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3259         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3260     }
3261
3262     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3263
3264     push @$cmd, '-daemonize';
3265
3266     if ($conf->{smbios1}) {
3267         my $smbios_conf = parse_smbios1($conf->{smbios1});
3268         if ($smbios_conf->{base64}) {
3269             # Do not pass base64 flag to qemu
3270             delete $smbios_conf->{base64};
3271             my $smbios_string = "";
3272             foreach my $key (keys %$smbios_conf) {
3273                 my $value;
3274                 if ($key eq "uuid") {
3275                     $value = $smbios_conf->{uuid}
3276                 } else {
3277                     $value = decode_base64($smbios_conf->{$key});
3278                 }
3279                 # qemu accepts any binary data, only commas need escaping by double comma
3280                 $value =~ s/,/,,/g;
3281                 $smbios_string .= "," . $key . "=" . $value if $value;
3282             }
3283             push @$cmd, '-smbios', "type=1" . $smbios_string;
3284         } else {
3285             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3286         }
3287     }
3288
3289     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3290         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3291         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3292
3293         my ($path, $format);
3294         my $read_only_str = '';
3295         if (my $efidisk = $conf->{efidisk0}) {
3296             my $d = parse_drive('efidisk0', $efidisk);
3297             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3298             $format = $d->{format};
3299             if ($storeid) {
3300                 $path = PVE::Storage::path($storecfg, $d->{file});
3301                 if (!defined($format)) {
3302                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3303                     $format = qemu_img_format($scfg, $volname);
3304                 }
3305             } else {
3306                 $path = $d->{file};
3307                 die "efidisk format must be specified\n"
3308                     if !defined($format);
3309             }
3310
3311             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3312         } else {
3313             warn "no efidisk configured! Using temporary efivars disk.\n";
3314             $path = "/tmp/$vmid-ovmf.fd";
3315             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3316             $format = 'raw';
3317         }
3318
3319         my $size_str = "";
3320
3321         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3322             $size_str = ",size=" . (-s $ovmf_vars);
3323         }
3324
3325         # on slower ceph clusters, booting without cache on efidisk can take a while, see #3329
3326         my $cache = $path =~ m/^rbd:/ ? ',cache=writeback' : '';
3327
3328         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3329         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3330     }
3331
3332     # load q35 config
3333     if ($q35) {
3334         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3335         if (min_version($machine_version, 4, 0)) {
3336             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3337         } else {
3338             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3339         }
3340     }
3341
3342     if ($conf->{vmgenid}) {
3343         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3344     }
3345
3346     # add usb controllers
3347     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3348         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3349     push @$devices, @usbcontrollers if @usbcontrollers;
3350     my $vga = parse_vga($conf->{vga});
3351
3352     my $qxlnum = vga_conf_has_spice($conf->{vga});
3353     $vga->{type} = 'qxl' if $qxlnum;
3354
3355     if (!$vga->{type}) {
3356         if ($arch eq 'aarch64') {
3357             $vga->{type} = 'virtio';
3358         } elsif (min_version($machine_version, 2, 9)) {
3359             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3360         } else {
3361             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3362         }
3363     }
3364
3365     # enable absolute mouse coordinates (needed by vnc)
3366     my $tablet;
3367     if (defined($conf->{tablet})) {
3368         $tablet = $conf->{tablet};
3369     } else {
3370         $tablet = $defaults->{tablet};
3371         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3372         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3373     }
3374
3375     if ($tablet) {
3376         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3377         my $kbd = print_keyboarddevice_full($conf, $arch);
3378         push @$devices, '-device', $kbd if defined($kbd);
3379     }
3380
3381     my $bootorder = device_bootorder($conf);
3382
3383     # host pci device passthrough
3384     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3385         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3386
3387     # usb devices
3388     my $usb_dev_features = {};
3389     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3390
3391     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3392         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3393     push @$devices, @usbdevices if @usbdevices;
3394
3395     # serial devices
3396     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3397         if (my $path = $conf->{"serial$i"}) {
3398             if ($path eq 'socket') {
3399                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3400                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3401                 # On aarch64, serial0 is the UART device. Qemu only allows
3402                 # connecting UART devices via the '-serial' command line, as
3403                 # the device has a fixed slot on the hardware...
3404                 if ($arch eq 'aarch64' && $i == 0) {
3405                     push @$devices, '-serial', "chardev:serial$i";
3406                 } else {
3407                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3408                 }
3409             } else {
3410                 die "no such serial device\n" if ! -c $path;
3411                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3412                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3413             }
3414         }
3415     }
3416
3417     # parallel devices
3418     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3419         if (my $path = $conf->{"parallel$i"}) {
3420             die "no such parallel device\n" if ! -c $path;
3421             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3422             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3423             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3424         }
3425     }
3426
3427     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3428         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3429         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3430         push @$devices, @$audio_devs;
3431     }
3432
3433     my $sockets = 1;
3434     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3435     $sockets = $conf->{sockets} if  $conf->{sockets};
3436
3437     my $cores = $conf->{cores} || 1;
3438
3439     my $maxcpus = $sockets * $cores;
3440
3441     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3442
3443     my $allowed_vcpus = $cpuinfo->{cpus};
3444
3445     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3446         if ($allowed_vcpus < $maxcpus);
3447
3448     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3449
3450         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3451         for (my $i = 2; $i <= $vcpus; $i++)  {
3452             my $cpustr = print_cpu_device($conf,$i);
3453             push @$cmd, '-device', $cpustr;
3454         }
3455
3456     } else {
3457
3458         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3459     }
3460     push @$cmd, '-nodefaults';
3461
3462     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3463
3464     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3465
3466     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3467
3468     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3469         push @$devices, '-device', print_vga_device(
3470             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3471         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3472         push @$cmd,  '-vnc', "unix:$socket,password=on";
3473     } else {
3474         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3475         push @$cmd, '-nographic';
3476     }
3477
3478     # time drift fix
3479     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3480     my $useLocaltime = $conf->{localtime};
3481
3482     if ($winversion >= 5) { # windows
3483         $useLocaltime = 1 if !defined($conf->{localtime});
3484
3485         # use time drift fix when acpi is enabled
3486         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3487             $tdf = 1 if !defined($conf->{tdf});
3488         }
3489     }
3490
3491     if ($winversion >= 6) {
3492         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3493         push @$cmd, '-no-hpet';
3494     }
3495
3496     push @$rtcFlags, 'driftfix=slew' if $tdf;
3497
3498     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3499         push @$rtcFlags, "base=$conf->{startdate}";
3500     } elsif ($useLocaltime) {
3501         push @$rtcFlags, 'base=localtime';
3502     }
3503
3504     if ($forcecpu) {
3505         push @$cmd, '-cpu', $forcecpu;
3506     } else {
3507         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3508     }
3509
3510     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3511
3512     push @$cmd, '-S' if $conf->{freeze};
3513
3514     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3515
3516     my $guest_agent = parse_guest_agent($conf);
3517
3518     if ($guest_agent->{enabled}) {
3519         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3520         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3521
3522         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3523             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3524             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3525             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3526         } elsif ($guest_agent->{type} eq 'isa') {
3527             push @$devices, '-device', "isa-serial,chardev=qga0";
3528         }
3529     }
3530
3531     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3532     if ($rng && $version_guard->(4, 1, 2)) {
3533         check_rng_source($rng->{source});
3534
3535         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3536         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3537         my $limiter_str = "";
3538         if ($max_bytes) {
3539             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3540         }
3541
3542         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3543         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3544         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3545     }
3546
3547     my $spice_port;
3548
3549     if ($qxlnum) {
3550         if ($qxlnum > 1) {
3551             if ($winversion){
3552                 for (my $i = 1; $i < $qxlnum; $i++){
3553                     push @$devices, '-device', print_vga_device(
3554                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3555                 }
3556             } else {
3557                 # assume other OS works like Linux
3558                 my ($ram, $vram) = ("134217728", "67108864");
3559                 if ($vga->{memory}) {
3560                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3561                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3562                 }
3563                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3564                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3565             }
3566         }
3567
3568         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3569
3570         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3571         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3572         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3573
3574         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3575         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3576         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3577
3578         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3579         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3580
3581         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3582         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3583         if ($spice_enhancement->{foldersharing}) {
3584             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3585             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3586         }
3587
3588         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3589         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3590             if $spice_enhancement->{videostreaming};
3591
3592         push @$devices, '-spice', "$spice_opts";
3593     }
3594
3595     # enable balloon by default, unless explicitly disabled
3596     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3597         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3598         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3599     }
3600
3601     if ($conf->{watchdog}) {
3602         my $wdopts = parse_watchdog($conf->{watchdog});
3603         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3604         my $watchdog = $wdopts->{model} || 'i6300esb';
3605         push @$devices, '-device', "$watchdog$pciaddr";
3606         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3607     }
3608
3609     my $vollist = [];
3610     my $scsicontroller = {};
3611     my $ahcicontroller = {};
3612     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3613
3614     # Add iscsi initiator name if available
3615     if (my $initiator = get_initiator_name()) {
3616         push @$devices, '-iscsi', "initiator-name=$initiator";
3617     }
3618
3619     PVE::QemuConfig->foreach_volume($conf, sub {
3620         my ($ds, $drive) = @_;
3621
3622         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3623             check_volume_storage_type($storecfg, $drive->{file});
3624             push @$vollist, $drive->{file};
3625         }
3626
3627         # ignore efidisk here, already added in bios/fw handling code above
3628         return if $drive->{interface} eq 'efidisk';
3629
3630         $use_virtio = 1 if $ds =~ m/^virtio/;
3631
3632         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3633
3634         if ($drive->{interface} eq 'virtio'){
3635            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3636         }
3637
3638         if ($drive->{interface} eq 'scsi') {
3639
3640             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3641
3642             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3643                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3644
3645             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3646             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3647
3648             my $iothread = '';
3649             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3650                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3651                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3652             } elsif ($drive->{iothread}) {
3653                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3654             }
3655
3656             my $queues = '';
3657             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3658                 $queues = ",num_queues=$drive->{queues}";
3659             }
3660
3661             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3662                 if !$scsicontroller->{$controller};
3663             $scsicontroller->{$controller}=1;
3664         }
3665
3666         if ($drive->{interface} eq 'sata') {
3667             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3668             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3669             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3670                 if !$ahcicontroller->{$controller};
3671             $ahcicontroller->{$controller}=1;
3672         }
3673
3674         my $pbs_conf = $pbs_backing->{$ds};
3675         my $pbs_name = undef;
3676         if ($pbs_conf) {
3677             $pbs_name = "drive-$ds-pbs";
3678             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3679         }
3680
3681         my $drive_cmd = print_drive_commandline_full(
3682             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3683
3684         # extra protection for templates, but SATA and IDE don't support it..
3685         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3686
3687         push @$devices, '-drive',$drive_cmd;
3688         push @$devices, '-device', print_drivedevice_full(
3689             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3690     });
3691
3692     for (my $i = 0; $i < $MAX_NETS; $i++) {
3693         my $netname = "net$i";
3694
3695         next if !$conf->{$netname};
3696         my $d = parse_net($conf->{$netname});
3697         next if !$d;
3698
3699         $use_virtio = 1 if $d->{model} eq 'virtio';
3700
3701         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3702
3703         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3704         push @$devices, '-netdev', $netdevfull;
3705
3706         my $netdevicefull = print_netdevice_full(
3707             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3708
3709         push @$devices, '-device', $netdevicefull;
3710     }
3711
3712     if ($conf->{ivshmem}) {
3713         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3714
3715         my $bus;
3716         if ($q35) {
3717             $bus = print_pcie_addr("ivshmem");
3718         } else {
3719             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3720         }
3721
3722         my $ivshmem_name = $ivshmem->{name} // $vmid;
3723         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3724
3725         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3726         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3727             .",size=$ivshmem->{size}M";
3728     }
3729
3730     # pci.4 is nested in pci.1
3731     $bridges->{1} = 1 if $bridges->{4};
3732
3733     if (!$q35) {
3734         # add pci bridges
3735         if (min_version($machine_version, 2, 3)) {
3736            $bridges->{1} = 1;
3737            $bridges->{2} = 1;
3738         }
3739
3740         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3741
3742     }
3743
3744     for my $k (sort {$b cmp $a} keys %$bridges) {
3745         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3746
3747         my $k_name = $k;
3748         if ($k == 2 && $legacy_igd) {
3749             $k_name = "$k-igd";
3750         }
3751         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3752
3753         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3754         if ($q35) {
3755             # add after -readconfig pve-q35.cfg
3756             splice @$devices, 2, 0, '-device', $devstr;
3757         } else {
3758             unshift @$devices, '-device', $devstr if $k > 0;
3759         }
3760     }
3761
3762     if (!$kvm) {
3763         push @$machineFlags, 'accel=tcg';
3764     }
3765
3766     my $machine_type_min = $machine_type;
3767     if ($add_pve_version) {
3768         $machine_type_min =~ s/\+pve\d+$//;
3769         $machine_type_min .= "+pve$required_pve_version";
3770     }
3771     push @$machineFlags, "type=${machine_type_min}";
3772
3773     push @$cmd, @$devices;
3774     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3775     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3776     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3777
3778     if (my $vmstate = $conf->{vmstate}) {
3779         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3780         push @$vollist, $vmstate;
3781         push @$cmd, '-loadstate', $statepath;
3782         print "activating and using '$vmstate' as vmstate\n";
3783     }
3784
3785     # add custom args
3786     if ($conf->{args}) {
3787         my $aa = PVE::Tools::split_args($conf->{args});
3788         push @$cmd, @$aa;
3789     }
3790
3791     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3792 }
3793
3794 sub check_rng_source {
3795     my ($source) = @_;
3796
3797     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3798     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3799         if ! -e $source;
3800
3801     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3802     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3803         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3804         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3805         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3806             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3807             ." to the host.\n";
3808     }
3809 }
3810
3811 sub spice_port {
3812     my ($vmid) = @_;
3813
3814     my $res = mon_cmd($vmid, 'query-spice');
3815
3816     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3817 }
3818
3819 sub vm_devices_list {
3820     my ($vmid) = @_;
3821
3822     my $res = mon_cmd($vmid, 'query-pci');
3823     my $devices_to_check = [];
3824     my $devices = {};
3825     foreach my $pcibus (@$res) {
3826         push @$devices_to_check, @{$pcibus->{devices}},
3827     }
3828
3829     while (@$devices_to_check) {
3830         my $to_check = [];
3831         for my $d (@$devices_to_check) {
3832             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3833             next if !$d->{'pci_bridge'};
3834
3835             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3836             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3837         }
3838         $devices_to_check = $to_check;
3839     }
3840
3841     my $resblock = mon_cmd($vmid, 'query-block');
3842     foreach my $block (@$resblock) {
3843         if($block->{device} =~ m/^drive-(\S+)/){
3844                 $devices->{$1} = 1;
3845         }
3846     }
3847
3848     my $resmice = mon_cmd($vmid, 'query-mice');
3849     foreach my $mice (@$resmice) {
3850         if ($mice->{name} eq 'QEMU HID Tablet') {
3851             $devices->{tablet} = 1;
3852             last;
3853         }
3854     }
3855
3856     # for usb devices there is no query-usb
3857     # but we can iterate over the entries in
3858     # qom-list path=/machine/peripheral
3859     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3860     foreach my $per (@$resperipheral) {
3861         if ($per->{name} =~ m/^usb\d+$/) {
3862             $devices->{$per->{name}} = 1;
3863         }
3864     }
3865
3866     return $devices;
3867 }
3868
3869 sub vm_deviceplug {
3870     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3871
3872     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3873
3874     my $devices_list = vm_devices_list($vmid);
3875     return 1 if defined($devices_list->{$deviceid});
3876
3877     # add PCI bridge if we need it for the device
3878     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3879
3880     if ($deviceid eq 'tablet') {
3881
3882         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3883
3884     } elsif ($deviceid eq 'keyboard') {
3885
3886         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3887
3888     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3889
3890         die "usb hotplug currently not reliable\n";
3891         # since we can't reliably hot unplug all added usb devices and usb
3892         # passthrough breaks live migration we disable usb hotplugging for now
3893         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3894
3895     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3896
3897         qemu_iothread_add($vmid, $deviceid, $device);
3898
3899         qemu_driveadd($storecfg, $vmid, $device);
3900         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3901
3902         qemu_deviceadd($vmid, $devicefull);
3903         eval { qemu_deviceaddverify($vmid, $deviceid); };
3904         if (my $err = $@) {
3905             eval { qemu_drivedel($vmid, $deviceid); };
3906             warn $@ if $@;
3907             die $err;
3908         }
3909
3910     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3911
3912
3913         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3914         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3915         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3916
3917         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3918
3919         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3920             qemu_iothread_add($vmid, $deviceid, $device);
3921             $devicefull .= ",iothread=iothread-$deviceid";
3922         }
3923
3924         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3925             $devicefull .= ",num_queues=$device->{queues}";
3926         }
3927
3928         qemu_deviceadd($vmid, $devicefull);
3929         qemu_deviceaddverify($vmid, $deviceid);
3930
3931     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3932
3933         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3934         qemu_driveadd($storecfg, $vmid, $device);
3935
3936         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3937         eval { qemu_deviceadd($vmid, $devicefull); };
3938         if (my $err = $@) {
3939             eval { qemu_drivedel($vmid, $deviceid); };
3940             warn $@ if $@;
3941             die $err;
3942         }
3943
3944     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3945
3946         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3947
3948         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3949         my $use_old_bios_files = undef;
3950         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3951
3952         my $netdevicefull = print_netdevice_full(
3953             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3954         qemu_deviceadd($vmid, $netdevicefull);
3955         eval {
3956             qemu_deviceaddverify($vmid, $deviceid);
3957             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3958         };
3959         if (my $err = $@) {
3960             eval { qemu_netdevdel($vmid, $deviceid); };
3961             warn $@ if $@;
3962             die $err;
3963         }
3964
3965     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3966
3967         my $bridgeid = $2;
3968         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3969         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3970
3971         qemu_deviceadd($vmid, $devicefull);
3972         qemu_deviceaddverify($vmid, $deviceid);
3973
3974     } else {
3975         die "can't hotplug device '$deviceid'\n";
3976     }
3977
3978     return 1;
3979 }
3980
3981 # fixme: this should raise exceptions on error!
3982 sub vm_deviceunplug {
3983     my ($vmid, $conf, $deviceid) = @_;
3984
3985     my $devices_list = vm_devices_list($vmid);
3986     return 1 if !defined($devices_list->{$deviceid});
3987
3988     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3989     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3990
3991     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3992
3993         qemu_devicedel($vmid, $deviceid);
3994
3995     } elsif ($deviceid =~ m/^usb\d+$/) {
3996
3997         die "usb hotplug currently not reliable\n";
3998         # when unplugging usb devices this way, there may be remaining usb
3999         # controllers/hubs so we disable it for now
4000         #qemu_devicedel($vmid, $deviceid);
4001         #qemu_devicedelverify($vmid, $deviceid);
4002
4003     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4004
4005         qemu_devicedel($vmid, $deviceid);
4006         qemu_devicedelverify($vmid, $deviceid);
4007         qemu_drivedel($vmid, $deviceid);
4008         qemu_iothread_del($conf, $vmid, $deviceid);
4009
4010     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4011
4012         qemu_devicedel($vmid, $deviceid);
4013         qemu_devicedelverify($vmid, $deviceid);
4014         qemu_iothread_del($conf, $vmid, $deviceid);
4015
4016     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4017
4018         qemu_devicedel($vmid, $deviceid);
4019         qemu_drivedel($vmid, $deviceid);
4020         qemu_deletescsihw($conf, $vmid, $deviceid);
4021
4022     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4023
4024         qemu_devicedel($vmid, $deviceid);
4025         qemu_devicedelverify($vmid, $deviceid);
4026         qemu_netdevdel($vmid, $deviceid);
4027
4028     } else {
4029         die "can't unplug device '$deviceid'\n";
4030     }
4031
4032     return 1;
4033 }
4034
4035 sub qemu_deviceadd {
4036     my ($vmid, $devicefull) = @_;
4037
4038     $devicefull = "driver=".$devicefull;
4039     my %options =  split(/[=,]/, $devicefull);
4040
4041     mon_cmd($vmid, "device_add" , %options);
4042 }
4043
4044 sub qemu_devicedel {
4045     my ($vmid, $deviceid) = @_;
4046
4047     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4048 }
4049
4050 sub qemu_iothread_add {
4051     my($vmid, $deviceid, $device) = @_;
4052
4053     if ($device->{iothread}) {
4054         my $iothreads = vm_iothreads_list($vmid);
4055         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4056     }
4057 }
4058
4059 sub qemu_iothread_del {
4060     my($conf, $vmid, $deviceid) = @_;
4061
4062     my $confid = $deviceid;
4063     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4064         $confid = 'scsi' . $1;
4065     }
4066     my $device = parse_drive($confid, $conf->{$confid});
4067     if ($device->{iothread}) {
4068         my $iothreads = vm_iothreads_list($vmid);
4069         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4070     }
4071 }
4072
4073 sub qemu_objectadd {
4074     my($vmid, $objectid, $qomtype) = @_;
4075
4076     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4077
4078     return 1;
4079 }
4080
4081 sub qemu_objectdel {
4082     my($vmid, $objectid) = @_;
4083
4084     mon_cmd($vmid, "object-del", id => $objectid);
4085
4086     return 1;
4087 }
4088
4089 sub qemu_driveadd {
4090     my ($storecfg, $vmid, $device) = @_;
4091
4092     my $kvmver = get_running_qemu_version($vmid);
4093     my $io_uring = min_version($kvmver, 6, 0);
4094     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4095     $drive =~ s/\\/\\\\/g;
4096     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4097
4098     # If the command succeeds qemu prints: "OK"
4099     return 1 if $ret =~ m/OK/s;
4100
4101     die "adding drive failed: $ret\n";
4102 }
4103
4104 sub qemu_drivedel {
4105     my($vmid, $deviceid) = @_;
4106
4107     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4108     $ret =~ s/^\s+//;
4109
4110     return 1 if $ret eq "";
4111
4112     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4113     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4114
4115     die "deleting drive $deviceid failed : $ret\n";
4116 }
4117
4118 sub qemu_deviceaddverify {
4119     my ($vmid, $deviceid) = @_;
4120
4121     for (my $i = 0; $i <= 5; $i++) {
4122          my $devices_list = vm_devices_list($vmid);
4123          return 1 if defined($devices_list->{$deviceid});
4124          sleep 1;
4125     }
4126
4127     die "error on hotplug device '$deviceid'\n";
4128 }
4129
4130
4131 sub qemu_devicedelverify {
4132     my ($vmid, $deviceid) = @_;
4133
4134     # need to verify that the device is correctly removed as device_del
4135     # is async and empty return is not reliable
4136
4137     for (my $i = 0; $i <= 5; $i++) {
4138          my $devices_list = vm_devices_list($vmid);
4139          return 1 if !defined($devices_list->{$deviceid});
4140          sleep 1;
4141     }
4142
4143     die "error on hot-unplugging device '$deviceid'\n";
4144 }
4145
4146 sub qemu_findorcreatescsihw {
4147     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4148
4149     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4150
4151     my $scsihwid="$controller_prefix$controller";
4152     my $devices_list = vm_devices_list($vmid);
4153
4154     if(!defined($devices_list->{$scsihwid})) {
4155         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4156     }
4157
4158     return 1;
4159 }
4160
4161 sub qemu_deletescsihw {
4162     my ($conf, $vmid, $opt) = @_;
4163
4164     my $device = parse_drive($opt, $conf->{$opt});
4165
4166     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4167         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4168         return 1;
4169     }
4170
4171     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4172
4173     my $devices_list = vm_devices_list($vmid);
4174     foreach my $opt (keys %{$devices_list}) {
4175         if (is_valid_drivename($opt)) {
4176             my $drive = parse_drive($opt, $conf->{$opt});
4177             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4178                 return 1;
4179             }
4180         }
4181     }
4182
4183     my $scsihwid="scsihw$controller";
4184
4185     vm_deviceunplug($vmid, $conf, $scsihwid);
4186
4187     return 1;
4188 }
4189
4190 sub qemu_add_pci_bridge {
4191     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4192
4193     my $bridges = {};
4194
4195     my $bridgeid;
4196
4197     print_pci_addr($device, $bridges, $arch, $machine_type);
4198
4199     while (my ($k, $v) = each %$bridges) {
4200         $bridgeid = $k;
4201     }
4202     return 1 if !defined($bridgeid) || $bridgeid < 1;
4203
4204     my $bridge = "pci.$bridgeid";
4205     my $devices_list = vm_devices_list($vmid);
4206
4207     if (!defined($devices_list->{$bridge})) {
4208         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4209     }
4210
4211     return 1;
4212 }
4213
4214 sub qemu_set_link_status {
4215     my ($vmid, $device, $up) = @_;
4216
4217     mon_cmd($vmid, "set_link", name => $device,
4218                up => $up ? JSON::true : JSON::false);
4219 }
4220
4221 sub qemu_netdevadd {
4222     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4223
4224     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4225     my %options =  split(/[=,]/, $netdev);
4226
4227     if (defined(my $vhost = $options{vhost})) {
4228         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4229     }
4230
4231     if (defined(my $queues = $options{queues})) {
4232         $options{queues} = $queues + 0;
4233     }
4234
4235     mon_cmd($vmid, "netdev_add",  %options);
4236     return 1;
4237 }
4238
4239 sub qemu_netdevdel {
4240     my ($vmid, $deviceid) = @_;
4241
4242     mon_cmd($vmid, "netdev_del", id => $deviceid);
4243 }
4244
4245 sub qemu_usb_hotplug {
4246     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4247
4248     return if !$device;
4249
4250     # remove the old one first
4251     vm_deviceunplug($vmid, $conf, $deviceid);
4252
4253     # check if xhci controller is necessary and available
4254     if ($device->{usb3}) {
4255
4256         my $devicelist = vm_devices_list($vmid);
4257
4258         if (!$devicelist->{xhci}) {
4259             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4260             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4261         }
4262     }
4263     my $d = parse_usb_device($device->{host});
4264     $d->{usb3} = $device->{usb3};
4265
4266     # add the new one
4267     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4268 }
4269
4270 sub qemu_cpu_hotplug {
4271     my ($vmid, $conf, $vcpus) = @_;
4272
4273     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4274
4275     my $sockets = 1;
4276     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4277     $sockets = $conf->{sockets} if  $conf->{sockets};
4278     my $cores = $conf->{cores} || 1;
4279     my $maxcpus = $sockets * $cores;
4280
4281     $vcpus = $maxcpus if !$vcpus;
4282
4283     die "you can't add more vcpus than maxcpus\n"
4284         if $vcpus > $maxcpus;
4285
4286     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4287
4288     if ($vcpus < $currentvcpus) {
4289
4290         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4291
4292             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4293                 qemu_devicedel($vmid, "cpu$i");
4294                 my $retry = 0;
4295                 my $currentrunningvcpus = undef;
4296                 while (1) {
4297                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4298                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4299                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4300                     $retry++;
4301                     sleep 1;
4302                 }
4303                 #update conf after each succesfull cpu unplug
4304                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4305                 PVE::QemuConfig->write_config($vmid, $conf);
4306             }
4307         } else {
4308             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4309         }
4310
4311         return;
4312     }
4313
4314     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4315     die "vcpus in running vm does not match its configuration\n"
4316         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4317
4318     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4319
4320         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4321             my $cpustr = print_cpu_device($conf, $i);
4322             qemu_deviceadd($vmid, $cpustr);
4323
4324             my $retry = 0;
4325             my $currentrunningvcpus = undef;
4326             while (1) {
4327                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4328                 last if scalar(@{$currentrunningvcpus}) == $i;
4329                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4330                 sleep 1;
4331                 $retry++;
4332             }
4333             #update conf after each succesfull cpu hotplug
4334             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4335             PVE::QemuConfig->write_config($vmid, $conf);
4336         }
4337     } else {
4338
4339         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4340             mon_cmd($vmid, "cpu-add", id => int($i));
4341         }
4342     }
4343 }
4344
4345 sub qemu_block_set_io_throttle {
4346     my ($vmid, $deviceid,
4347         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4348         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4349         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4350         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4351
4352     return if !check_running($vmid) ;
4353
4354     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4355         bps => int($bps),
4356         bps_rd => int($bps_rd),
4357         bps_wr => int($bps_wr),
4358         iops => int($iops),
4359         iops_rd => int($iops_rd),
4360         iops_wr => int($iops_wr),
4361         bps_max => int($bps_max),
4362         bps_rd_max => int($bps_rd_max),
4363         bps_wr_max => int($bps_wr_max),
4364         iops_max => int($iops_max),
4365         iops_rd_max => int($iops_rd_max),
4366         iops_wr_max => int($iops_wr_max),
4367         bps_max_length => int($bps_max_length),
4368         bps_rd_max_length => int($bps_rd_max_length),
4369         bps_wr_max_length => int($bps_wr_max_length),
4370         iops_max_length => int($iops_max_length),
4371         iops_rd_max_length => int($iops_rd_max_length),
4372         iops_wr_max_length => int($iops_wr_max_length),
4373     );
4374
4375 }
4376
4377 sub qemu_block_resize {
4378     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4379
4380     my $running = check_running($vmid);
4381
4382     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4383
4384     return if !$running;
4385
4386     my $padding = (1024 - $size % 1024) % 1024;
4387     $size = $size + $padding;
4388
4389     mon_cmd(
4390         $vmid,
4391         "block_resize",
4392         device => $deviceid,
4393         size => int($size),
4394         timeout => 60,
4395     );
4396 }
4397
4398 sub qemu_volume_snapshot {
4399     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4400
4401     my $running = check_running($vmid);
4402
4403     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4404         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4405     } else {
4406         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4407     }
4408 }
4409
4410 sub qemu_volume_snapshot_delete {
4411     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4412
4413     my $running = check_running($vmid);
4414
4415     if($running) {
4416
4417         $running = undef;
4418         my $conf = PVE::QemuConfig->load_config($vmid);
4419         PVE::QemuConfig->foreach_volume($conf, sub {
4420             my ($ds, $drive) = @_;
4421             $running = 1 if $drive->{file} eq $volid;
4422         });
4423     }
4424
4425     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4426         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4427     } else {
4428         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4429     }
4430 }
4431
4432 sub set_migration_caps {
4433     my ($vmid, $savevm) = @_;
4434
4435     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4436
4437     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4438     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4439
4440     my $cap_ref = [];
4441
4442     my $enabled_cap = {
4443         "auto-converge" => 1,
4444         "xbzrle" => 1,
4445         "x-rdma-pin-all" => 0,
4446         "zero-blocks" => 0,
4447         "compress" => 0,
4448         "dirty-bitmaps" => $dirty_bitmaps,
4449     };
4450
4451     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4452
4453     for my $supported_capability (@$supported_capabilities) {
4454         push @$cap_ref, {
4455             capability => $supported_capability->{capability},
4456             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4457         };
4458     }
4459
4460     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4461 }
4462
4463 sub foreach_volid {
4464     my ($conf, $func, @param) = @_;
4465
4466     my $volhash = {};
4467
4468     my $test_volid = sub {
4469         my ($key, $drive, $snapname) = @_;
4470
4471         my $volid = $drive->{file};
4472         return if !$volid;
4473
4474         $volhash->{$volid}->{cdrom} //= 1;
4475         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4476
4477         my $replicate = $drive->{replicate} // 1;
4478         $volhash->{$volid}->{replicate} //= 0;
4479         $volhash->{$volid}->{replicate} = 1 if $replicate;
4480
4481         $volhash->{$volid}->{shared} //= 0;
4482         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4483
4484         $volhash->{$volid}->{referenced_in_config} //= 0;
4485         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4486
4487         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4488             if defined($snapname);
4489
4490         my $size = $drive->{size};
4491         $volhash->{$volid}->{size} //= $size if $size;
4492
4493         $volhash->{$volid}->{is_vmstate} //= 0;
4494         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4495
4496         $volhash->{$volid}->{is_unused} //= 0;
4497         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4498
4499         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4500     };
4501
4502     my $include_opts = {
4503         extra_keys => ['vmstate'],
4504         include_unused => 1,
4505     };
4506
4507     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4508     foreach my $snapname (keys %{$conf->{snapshots}}) {
4509         my $snap = $conf->{snapshots}->{$snapname};
4510         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4511     }
4512
4513     foreach my $volid (keys %$volhash) {
4514         &$func($volid, $volhash->{$volid}, @param);
4515     }
4516 }
4517
4518 my $fast_plug_option = {
4519     'lock' => 1,
4520     'name' => 1,
4521     'onboot' => 1,
4522     'shares' => 1,
4523     'startup' => 1,
4524     'description' => 1,
4525     'protection' => 1,
4526     'vmstatestorage' => 1,
4527     'hookscript' => 1,
4528     'tags' => 1,
4529 };
4530
4531 # hotplug changes in [PENDING]
4532 # $selection hash can be used to only apply specified options, for
4533 # example: { cores => 1 } (only apply changed 'cores')
4534 # $errors ref is used to return error messages
4535 sub vmconfig_hotplug_pending {
4536     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4537
4538     my $defaults = load_defaults();
4539     my $arch = get_vm_arch($conf);
4540     my $machine_type = get_vm_machine($conf, undef, $arch);
4541
4542     # commit values which do not have any impact on running VM first
4543     # Note: those option cannot raise errors, we we do not care about
4544     # $selection and always apply them.
4545
4546     my $add_error = sub {
4547         my ($opt, $msg) = @_;
4548         $errors->{$opt} = "hotplug problem - $msg";
4549     };
4550
4551     my $changes = 0;
4552     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4553         if ($fast_plug_option->{$opt}) {
4554             $conf->{$opt} = $conf->{pending}->{$opt};
4555             delete $conf->{pending}->{$opt};
4556             $changes = 1;
4557         }
4558     }
4559
4560     if ($changes) {
4561         PVE::QemuConfig->write_config($vmid, $conf);
4562     }
4563
4564     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4565
4566     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4567     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4568     foreach my $opt (sort keys %$pending_delete_hash) {
4569         next if $selection && !$selection->{$opt};
4570         my $force = $pending_delete_hash->{$opt}->{force};
4571         eval {
4572             if ($opt eq 'hotplug') {
4573                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4574             } elsif ($opt eq 'tablet') {
4575                 die "skip\n" if !$hotplug_features->{usb};
4576                 if ($defaults->{tablet}) {
4577                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4578                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4579                         if $arch eq 'aarch64';
4580                 } else {
4581                     vm_deviceunplug($vmid, $conf, 'tablet');
4582                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4583                 }
4584             } elsif ($opt =~ m/^usb\d+/) {
4585                 die "skip\n";
4586                 # since we cannot reliably hot unplug usb devices we are disabling it
4587                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4588                 #vm_deviceunplug($vmid, $conf, $opt);
4589             } elsif ($opt eq 'vcpus') {
4590                 die "skip\n" if !$hotplug_features->{cpu};
4591                 qemu_cpu_hotplug($vmid, $conf, undef);
4592             } elsif ($opt eq 'balloon') {
4593                 # enable balloon device is not hotpluggable
4594                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4595                 # here we reset the ballooning value to memory
4596                 my $balloon = $conf->{memory} || $defaults->{memory};
4597                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4598             } elsif ($fast_plug_option->{$opt}) {
4599                 # do nothing
4600             } elsif ($opt =~ m/^net(\d+)$/) {
4601                 die "skip\n" if !$hotplug_features->{network};
4602                 vm_deviceunplug($vmid, $conf, $opt);
4603             } elsif (is_valid_drivename($opt)) {
4604                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4605                 vm_deviceunplug($vmid, $conf, $opt);
4606                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4607             } elsif ($opt =~ m/^memory$/) {
4608                 die "skip\n" if !$hotplug_features->{memory};
4609                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4610             } elsif ($opt eq 'cpuunits') {
4611                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4612             } elsif ($opt eq 'cpulimit') {
4613                 $cgroup->change_cpu_quota(-1, 100000);
4614             } else {
4615                 die "skip\n";
4616             }
4617         };
4618         if (my $err = $@) {
4619             &$add_error($opt, $err) if $err ne "skip\n";
4620         } else {
4621             delete $conf->{$opt};
4622             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4623         }
4624     }
4625
4626     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4627     $apply_pending_cloudinit = sub {
4628         return if $apply_pending_cloudinit_done; # once is enough
4629         $apply_pending_cloudinit_done = 1; # once is enough
4630
4631         my ($key, $value) = @_;
4632
4633         my @cloudinit_opts = keys %$confdesc_cloudinit;
4634         foreach my $opt (keys %{$conf->{pending}}) {
4635             next if !grep { $_ eq $opt } @cloudinit_opts;
4636             $conf->{$opt} = delete $conf->{pending}->{$opt};
4637         }
4638
4639         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4640         foreach my $opt (sort keys %$pending_delete_hash) {
4641             next if !grep { $_ eq $opt } @cloudinit_opts;
4642             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4643             delete $conf->{$opt};
4644         }
4645
4646         my $new_conf = { %$conf };
4647         $new_conf->{$key} = $value;
4648         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4649     };
4650
4651     foreach my $opt (keys %{$conf->{pending}}) {
4652         next if $selection && !$selection->{$opt};
4653         my $value = $conf->{pending}->{$opt};
4654         eval {
4655             if ($opt eq 'hotplug') {
4656                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4657             } elsif ($opt eq 'tablet') {
4658                 die "skip\n" if !$hotplug_features->{usb};
4659                 if ($value == 1) {
4660                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4661                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4662                         if $arch eq 'aarch64';
4663                 } elsif ($value == 0) {
4664                     vm_deviceunplug($vmid, $conf, 'tablet');
4665                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4666                 }
4667             } elsif ($opt =~ m/^usb\d+$/) {
4668                 die "skip\n";
4669                 # since we cannot reliably hot unplug usb devices we disable it for now
4670                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4671                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4672                 #die "skip\n" if !$d;
4673                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4674             } elsif ($opt eq 'vcpus') {
4675                 die "skip\n" if !$hotplug_features->{cpu};
4676                 qemu_cpu_hotplug($vmid, $conf, $value);
4677             } elsif ($opt eq 'balloon') {
4678                 # enable/disable balloning device is not hotpluggable
4679                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4680                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4681                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4682
4683                 # allow manual ballooning if shares is set to zero
4684                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4685                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4686                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4687                 }
4688             } elsif ($opt =~ m/^net(\d+)$/) {
4689                 # some changes can be done without hotplug
4690                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4691                                     $vmid, $opt, $value, $arch, $machine_type);
4692             } elsif (is_valid_drivename($opt)) {
4693                 die "skip\n" if $opt eq 'efidisk0';
4694                 # some changes can be done without hotplug
4695                 my $drive = parse_drive($opt, $value);
4696                 if (drive_is_cloudinit($drive)) {
4697                     &$apply_pending_cloudinit($opt, $value);
4698                 }
4699                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4700                                      $vmid, $opt, $value, $arch, $machine_type);
4701             } elsif ($opt =~ m/^memory$/) { #dimms
4702                 die "skip\n" if !$hotplug_features->{memory};
4703                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4704             } elsif ($opt eq 'cpuunits') {
4705                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4706             } elsif ($opt eq 'cpulimit') {
4707                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4708                 $cgroup->change_cpu_quota($cpulimit, 100000);
4709             } else {
4710                 die "skip\n";  # skip non-hot-pluggable options
4711             }
4712         };
4713         if (my $err = $@) {
4714             &$add_error($opt, $err) if $err ne "skip\n";
4715         } else {
4716             $conf->{$opt} = $value;
4717             delete $conf->{pending}->{$opt};
4718         }
4719     }
4720
4721     PVE::QemuConfig->write_config($vmid, $conf);
4722 }
4723
4724 sub try_deallocate_drive {
4725     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4726
4727     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4728         my $volid = $drive->{file};
4729         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4730             my $sid = PVE::Storage::parse_volume_id($volid);
4731             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4732
4733             # check if the disk is really unused
4734             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4735                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4736             PVE::Storage::vdisk_free($storecfg, $volid);
4737             return 1;
4738         } else {
4739             # If vm is not owner of this disk remove from config
4740             return 1;
4741         }
4742     }
4743
4744     return;
4745 }
4746
4747 sub vmconfig_delete_or_detach_drive {
4748     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4749
4750     my $drive = parse_drive($opt, $conf->{$opt});
4751
4752     my $rpcenv = PVE::RPCEnvironment::get();
4753     my $authuser = $rpcenv->get_user();
4754
4755     if ($force) {
4756         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4757         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4758     } else {
4759         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4760     }
4761 }
4762
4763
4764
4765 sub vmconfig_apply_pending {
4766     my ($vmid, $conf, $storecfg, $errors) = @_;
4767
4768     my $add_apply_error = sub {
4769         my ($opt, $msg) = @_;
4770         my $err_msg = "unable to apply pending change $opt : $msg";
4771         $errors->{$opt} = $err_msg;
4772         warn $err_msg;
4773     };
4774
4775     # cold plug
4776
4777     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4778     foreach my $opt (sort keys %$pending_delete_hash) {
4779         my $force = $pending_delete_hash->{$opt}->{force};
4780         eval {
4781             if ($opt =~ m/^unused/) {
4782                 die "internal error";
4783             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4784                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4785             }
4786         };
4787         if (my $err = $@) {
4788             $add_apply_error->($opt, $err);
4789         } else {
4790             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4791             delete $conf->{$opt};
4792         }
4793     }
4794
4795     PVE::QemuConfig->cleanup_pending($conf);
4796
4797     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4798         next if $opt eq 'delete'; # just to be sure
4799         eval {
4800             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4801                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4802             }
4803         };
4804         if (my $err = $@) {
4805             $add_apply_error->($opt, $err);
4806         } else {
4807             $conf->{$opt} = delete $conf->{pending}->{$opt};
4808         }
4809     }
4810
4811     # write all changes at once to avoid unnecessary i/o
4812     PVE::QemuConfig->write_config($vmid, $conf);
4813 }
4814
4815 sub vmconfig_update_net {
4816     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4817
4818     my $newnet = parse_net($value);
4819
4820     if ($conf->{$opt}) {
4821         my $oldnet = parse_net($conf->{$opt});
4822
4823         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4824             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4825             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4826             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4827
4828             # for non online change, we try to hot-unplug
4829             die "skip\n" if !$hotplug;
4830             vm_deviceunplug($vmid, $conf, $opt);
4831         } else {
4832
4833             die "internal error" if $opt !~ m/net(\d+)/;
4834             my $iface = "tap${vmid}i$1";
4835
4836             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4837                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4838                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4839                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4840                 PVE::Network::tap_unplug($iface);
4841
4842                 if ($have_sdn) {
4843                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4844                 } else {
4845                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4846                 }
4847             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4848                 # Rate can be applied on its own but any change above needs to
4849                 # include the rate in tap_plug since OVS resets everything.
4850                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4851             }
4852
4853             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4854                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4855             }
4856
4857             return 1;
4858         }
4859     }
4860
4861     if ($hotplug) {
4862         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4863     } else {
4864         die "skip\n";
4865     }
4866 }
4867
4868 sub vmconfig_update_disk {
4869     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4870
4871     my $drive = parse_drive($opt, $value);
4872
4873     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4874         my $media = $drive->{media} || 'disk';
4875         my $oldmedia = $old_drive->{media} || 'disk';
4876         die "unable to change media type\n" if $media ne $oldmedia;
4877
4878         if (!drive_is_cdrom($old_drive)) {
4879
4880             if ($drive->{file} ne $old_drive->{file}) {
4881
4882                 die "skip\n" if !$hotplug;
4883
4884                 # unplug and register as unused
4885                 vm_deviceunplug($vmid, $conf, $opt);
4886                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4887
4888             } else {
4889                 # update existing disk
4890
4891                 # skip non hotpluggable value
4892                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4893                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4894                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4895                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4896                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4897                     die "skip\n";
4898                 }
4899
4900                 # apply throttle
4901                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4902                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4903                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4904                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4905                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4906                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4907                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4908                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4909                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4910                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4911                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4912                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4913                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4914                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4915                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4916                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4917                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4918                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4919
4920                     qemu_block_set_io_throttle(
4921                         $vmid,"drive-$opt",
4922                         ($drive->{mbps} || 0)*1024*1024,
4923                         ($drive->{mbps_rd} || 0)*1024*1024,
4924                         ($drive->{mbps_wr} || 0)*1024*1024,
4925                          $drive->{iops} || 0,
4926                          $drive->{iops_rd} || 0,
4927                          $drive->{iops_wr} || 0,
4928                         ($drive->{mbps_max} || 0)*1024*1024,
4929                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4930                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4931                          $drive->{iops_max} || 0,
4932                          $drive->{iops_rd_max} || 0,
4933                          $drive->{iops_wr_max} || 0,
4934                          $drive->{bps_max_length} || 1,
4935                          $drive->{bps_rd_max_length} || 1,
4936                          $drive->{bps_wr_max_length} || 1,
4937                          $drive->{iops_max_length} || 1,
4938                          $drive->{iops_rd_max_length} || 1,
4939                          $drive->{iops_wr_max_length} || 1,
4940                     );
4941
4942                 }
4943
4944                 return 1;
4945             }
4946
4947         } else { # cdrom
4948
4949             if ($drive->{file} eq 'none') {
4950                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4951                 if (drive_is_cloudinit($old_drive)) {
4952                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4953                 }
4954             } else {
4955                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4956
4957                 # force eject if locked
4958                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4959
4960                 if ($path) {
4961                     mon_cmd($vmid, "blockdev-change-medium",
4962                         id => "$opt", filename => "$path");
4963                 }
4964             }
4965
4966             return 1;
4967         }
4968     }
4969
4970     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4971     # hotplug new disks
4972     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4973     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4974 }
4975
4976 # called in locked context by incoming migration
4977 sub vm_migrate_get_nbd_disks {
4978     my ($storecfg, $conf, $replicated_volumes) = @_;
4979
4980     my $local_volumes = {};
4981     PVE::QemuConfig->foreach_volume($conf, sub {
4982         my ($ds, $drive) = @_;
4983
4984         return if drive_is_cdrom($drive);
4985
4986         my $volid = $drive->{file};
4987
4988         return if !$volid;
4989
4990         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4991
4992         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4993         return if $scfg->{shared};
4994
4995         # replicated disks re-use existing state via bitmap
4996         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4997         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4998     });
4999     return $local_volumes;
5000 }
5001
5002 # called in locked context by incoming migration
5003 sub vm_migrate_alloc_nbd_disks {
5004     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5005
5006     my $format = undef;
5007
5008     my $nbd = {};
5009     foreach my $opt (sort keys %$source_volumes) {
5010         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5011
5012         if ($use_existing) {
5013             $nbd->{$opt}->{drivestr} = print_drive($drive);
5014             $nbd->{$opt}->{volid} = $volid;
5015             $nbd->{$opt}->{replicated} = 1;
5016             next;
5017         }
5018
5019         # If a remote storage is specified and the format of the original
5020         # volume is not available there, fall back to the default format.
5021         # Otherwise use the same format as the original.
5022         if (!$storagemap->{identity}) {
5023             $storeid = map_storage($storagemap, $storeid);
5024             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5025             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5026             my $fileFormat = qemu_img_format($scfg, $volname);
5027             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5028         } else {
5029             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5030             $format = qemu_img_format($scfg, $volname);
5031         }
5032
5033         my $size = $drive->{size} / 1024;
5034         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5035         my $newdrive = $drive;
5036         $newdrive->{format} = $format;
5037         $newdrive->{file} = $newvolid;
5038         my $drivestr = print_drive($newdrive);
5039         $nbd->{$opt}->{drivestr} = $drivestr;
5040         $nbd->{$opt}->{volid} = $newvolid;
5041     }
5042
5043     return $nbd;
5044 }
5045
5046 # see vm_start_nolock for parameters, additionally:
5047 # migrate_opts:
5048 #   storagemap = parsed storage map for allocating NBD disks
5049 sub vm_start {
5050     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5051
5052     return PVE::QemuConfig->lock_config($vmid, sub {
5053         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5054
5055         die "you can't start a vm if it's a template\n"
5056             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5057
5058         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5059         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5060
5061         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5062
5063         if ($has_backup_lock && $running) {
5064             # a backup is currently running, attempt to start the guest in the
5065             # existing QEMU instance
5066             return vm_resume($vmid);
5067         }
5068
5069         PVE::QemuConfig->check_lock($conf)
5070             if !($params->{skiplock} || $has_suspended_lock);
5071
5072         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5073
5074         die "VM $vmid already running\n" if $running;
5075
5076         if (my $storagemap = $migrate_opts->{storagemap}) {
5077             my $replicated = $migrate_opts->{replicated_volumes};
5078             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5079             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5080
5081             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5082                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5083             }
5084         }
5085
5086         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5087     });
5088 }
5089
5090
5091 # params:
5092 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5093 #   skiplock => 0/1, skip checking for config lock
5094 #   skiptemplate => 0/1, skip checking whether VM is template
5095 #   forcemachine => to force Qemu machine (rollback/migration)
5096 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5097 #   timeout => in seconds
5098 #   paused => start VM in paused state (backup)
5099 #   resume => resume from hibernation
5100 #   pbs-backing => {
5101 #      sata0 => {
5102 #         repository
5103 #         snapshot
5104 #         keyfile
5105 #         archive
5106 #      },
5107 #      virtio2 => ...
5108 #   }
5109 # migrate_opts:
5110 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5111 #   migratedfrom => source node
5112 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5113 #   network => CIDR of migration network
5114 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5115 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5116 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5117 sub vm_start_nolock {
5118     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5119
5120     my $statefile = $params->{statefile};
5121     my $resume = $params->{resume};
5122
5123     my $migratedfrom = $migrate_opts->{migratedfrom};
5124     my $migration_type = $migrate_opts->{type};
5125
5126     my $res = {};
5127
5128     # clean up leftover reboot request files
5129     eval { clear_reboot_request($vmid); };
5130     warn $@ if $@;
5131
5132     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5133         vmconfig_apply_pending($vmid, $conf, $storecfg);
5134         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5135     }
5136
5137     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5138
5139     my $defaults = load_defaults();
5140
5141     # set environment variable useful inside network script
5142     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5143
5144     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5145
5146     my $forcemachine = $params->{forcemachine};
5147     my $forcecpu = $params->{forcecpu};
5148     if ($resume) {
5149         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5150         $forcemachine = $conf->{runningmachine};
5151         $forcecpu = $conf->{runningcpu};
5152         print "Resuming suspended VM\n";
5153     }
5154
5155     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5156         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5157
5158     my $migration_ip;
5159     my $get_migration_ip = sub {
5160         my ($nodename) = @_;
5161
5162         return $migration_ip if defined($migration_ip);
5163
5164         my $cidr = $migrate_opts->{network};
5165
5166         if (!defined($cidr)) {
5167             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5168             $cidr = $dc_conf->{migration}->{network};
5169         }
5170
5171         if (defined($cidr)) {
5172             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5173
5174             die "could not get IP: no address configured on local " .
5175                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5176
5177             die "could not get IP: multiple addresses configured on local " .
5178                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5179
5180             $migration_ip = @$ips[0];
5181         }
5182
5183         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5184             if !defined($migration_ip);
5185
5186         return $migration_ip;
5187     };
5188
5189     my $migrate_uri;
5190     if ($statefile) {
5191         if ($statefile eq 'tcp') {
5192             my $localip = "localhost";
5193             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5194             my $nodename = nodename();
5195
5196             if (!defined($migration_type)) {
5197                 if (defined($datacenterconf->{migration}->{type})) {
5198                     $migration_type = $datacenterconf->{migration}->{type};
5199                 } else {
5200                     $migration_type = 'secure';
5201                 }
5202             }
5203
5204             if ($migration_type eq 'insecure') {
5205                 $localip = $get_migration_ip->($nodename);
5206                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5207             }
5208
5209             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5210             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5211             $migrate_uri = "tcp:${localip}:${migrate_port}";
5212             push @$cmd, '-incoming', $migrate_uri;
5213             push @$cmd, '-S';
5214
5215         } elsif ($statefile eq 'unix') {
5216             # should be default for secure migrations as a ssh TCP forward
5217             # tunnel is not deterministic reliable ready and fails regurarly
5218             # to set up in time, so use UNIX socket forwards
5219             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5220             unlink $socket_addr;
5221
5222             $migrate_uri = "unix:$socket_addr";
5223
5224             push @$cmd, '-incoming', $migrate_uri;
5225             push @$cmd, '-S';
5226
5227         } elsif (-e $statefile) {
5228             push @$cmd, '-loadstate', $statefile;
5229         } else {
5230             my $statepath = PVE::Storage::path($storecfg, $statefile);
5231             push @$vollist, $statefile;
5232             push @$cmd, '-loadstate', $statepath;
5233         }
5234     } elsif ($params->{paused}) {
5235         push @$cmd, '-S';
5236     }
5237
5238     # host pci devices
5239     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5240       my $d = parse_hostpci($conf->{"hostpci$i"});
5241       next if !$d;
5242       my $pcidevices = $d->{pciid};
5243       foreach my $pcidevice (@$pcidevices) {
5244             my $pciid = $pcidevice->{id};
5245
5246             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5247             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5248             die "no pci device info for device '$pciid'\n" if !$info;
5249
5250             if ($d->{mdev}) {
5251                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5252                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5253             } else {
5254                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5255                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5256                 die "can't reset PCI device '$pciid'\n"
5257                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5258             }
5259       }
5260     }
5261
5262     PVE::Storage::activate_volumes($storecfg, $vollist);
5263
5264     eval {
5265         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5266             outfunc => sub {}, errfunc => sub {});
5267     };
5268     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5269     # timeout should be more than enough here...
5270     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5271
5272     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5273                                               : $defaults->{cpuunits};
5274
5275     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5276     my %run_params = (
5277         timeout => $statefile ? undef : $start_timeout,
5278         umask => 0077,
5279         noerr => 1,
5280     );
5281
5282     # when migrating, prefix QEMU output so other side can pick up any
5283     # errors that might occur and show the user
5284     if ($migratedfrom) {
5285         $run_params{quiet} = 1;
5286         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5287     }
5288
5289     my %properties = (
5290         Slice => 'qemu.slice',
5291         KillMode => 'process',
5292         SendSIGKILL => 0,
5293         TimeoutStopUSec => ULONG_MAX, # infinity
5294     );
5295
5296     if (PVE::CGroup::cgroup_mode() == 2) {
5297         $properties{CPUWeight} = $cpuunits;
5298     } else {
5299         $properties{CPUShares} = $cpuunits;
5300     }
5301
5302     if (my $cpulimit = $conf->{cpulimit}) {
5303         $properties{CPUQuota} = int($cpulimit * 100);
5304     }
5305     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5306
5307     my $run_qemu = sub {
5308         PVE::Tools::run_fork sub {
5309             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5310
5311             my $exitcode = run_command($cmd, %run_params);
5312             die "QEMU exited with code $exitcode\n" if $exitcode;
5313         };
5314     };
5315
5316     if ($conf->{hugepages}) {
5317
5318         my $code = sub {
5319             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5320             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5321
5322             PVE::QemuServer::Memory::hugepages_mount();
5323             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5324
5325             eval { $run_qemu->() };
5326             if (my $err = $@) {
5327                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5328                     if !$conf->{keephugepages};
5329                 die $err;
5330             }
5331
5332             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5333                 if !$conf->{keephugepages};
5334         };
5335         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5336
5337     } else {
5338         eval { $run_qemu->() };
5339     }
5340
5341     if (my $err = $@) {
5342         # deactivate volumes if start fails
5343         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5344         die "start failed: $err";
5345     }
5346
5347     print "migration listens on $migrate_uri\n" if $migrate_uri;
5348     $res->{migrate_uri} = $migrate_uri;
5349
5350     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5351         eval { mon_cmd($vmid, "cont"); };
5352         warn $@ if $@;
5353     }
5354
5355     #start nbd server for storage migration
5356     if (my $nbd = $migrate_opts->{nbd}) {
5357         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5358
5359         my $migrate_storage_uri;
5360         # nbd_protocol_version > 0 for unix socket support
5361         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5362             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5363             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5364             $migrate_storage_uri = "nbd:unix:$socket_path";
5365         } else {
5366             my $nodename = nodename();
5367             my $localip = $get_migration_ip->($nodename);
5368             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5369             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5370
5371             mon_cmd($vmid, "nbd-server-start", addr => {
5372                 type => 'inet',
5373                 data => {
5374                     host => "${localip}",
5375                     port => "${storage_migrate_port}",
5376                 },
5377             });
5378             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5379             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5380         }
5381
5382         $res->{migrate_storage_uri} = $migrate_storage_uri;
5383
5384         foreach my $opt (sort keys %$nbd) {
5385             my $drivestr = $nbd->{$opt}->{drivestr};
5386             my $volid = $nbd->{$opt}->{volid};
5387             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5388             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5389             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5390             print "re-using replicated volume: $opt - $volid\n"
5391                 if $nbd->{$opt}->{replicated};
5392
5393             $res->{drives}->{$opt} = $nbd->{$opt};
5394             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5395         }
5396     }
5397
5398     if ($migratedfrom) {
5399         eval {
5400             set_migration_caps($vmid);
5401         };
5402         warn $@ if $@;
5403
5404         if ($spice_port) {
5405             print "spice listens on port $spice_port\n";
5406             $res->{spice_port} = $spice_port;
5407             if ($migrate_opts->{spice_ticket}) {
5408                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5409                     $migrate_opts->{spice_ticket});
5410                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5411             }
5412         }
5413
5414     } else {
5415         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5416             if !$statefile && $conf->{balloon};
5417
5418         foreach my $opt (keys %$conf) {
5419             next if $opt !~  m/^net\d+$/;
5420             my $nicconf = parse_net($conf->{$opt});
5421             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5422         }
5423     }
5424
5425     mon_cmd($vmid, 'qom-set',
5426                 path => "machine/peripheral/balloon0",
5427                 property => "guest-stats-polling-interval",
5428                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5429
5430     if ($resume) {
5431         print "Resumed VM, removing state\n";
5432         if (my $vmstate = $conf->{vmstate}) {
5433             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5434             PVE::Storage::vdisk_free($storecfg, $vmstate);
5435         }
5436         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5437         PVE::QemuConfig->write_config($vmid, $conf);
5438     }
5439
5440     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5441
5442     return $res;
5443 }
5444
5445 sub vm_commandline {
5446     my ($storecfg, $vmid, $snapname) = @_;
5447
5448     my $conf = PVE::QemuConfig->load_config($vmid);
5449     my $forcemachine;
5450     my $forcecpu;
5451
5452     if ($snapname) {
5453         my $snapshot = $conf->{snapshots}->{$snapname};
5454         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5455
5456         # check for machine or CPU overrides in snapshot
5457         $forcemachine = $snapshot->{runningmachine};
5458         $forcecpu = $snapshot->{runningcpu};
5459
5460         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5461
5462         $conf = $snapshot;
5463     }
5464
5465     my $defaults = load_defaults();
5466
5467     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5468         $forcemachine, $forcecpu);
5469
5470     return PVE::Tools::cmd2string($cmd);
5471 }
5472
5473 sub vm_reset {
5474     my ($vmid, $skiplock) = @_;
5475
5476     PVE::QemuConfig->lock_config($vmid, sub {
5477
5478         my $conf = PVE::QemuConfig->load_config($vmid);
5479
5480         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5481
5482         mon_cmd($vmid, "system_reset");
5483     });
5484 }
5485
5486 sub get_vm_volumes {
5487     my ($conf) = @_;
5488
5489     my $vollist = [];
5490     foreach_volid($conf, sub {
5491         my ($volid, $attr) = @_;
5492
5493         return if $volid =~ m|^/|;
5494
5495         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5496         return if !$sid;
5497
5498         push @$vollist, $volid;
5499     });
5500
5501     return $vollist;
5502 }
5503
5504 sub vm_stop_cleanup {
5505     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5506
5507     eval {
5508
5509         if (!$keepActive) {
5510             my $vollist = get_vm_volumes($conf);
5511             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5512         }
5513
5514         foreach my $ext (qw(mon qmp pid vnc qga)) {
5515             unlink "/var/run/qemu-server/${vmid}.$ext";
5516         }
5517
5518         if ($conf->{ivshmem}) {
5519             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5520             # just delete it for now, VMs which have this already open do not
5521             # are affected, but new VMs will get a separated one. If this
5522             # becomes an issue we either add some sort of ref-counting or just
5523             # add a "don't delete on stop" flag to the ivshmem format.
5524             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5525         }
5526
5527         foreach my $key (keys %$conf) {
5528             next if $key !~ m/^hostpci(\d+)$/;
5529             my $hostpciindex = $1;
5530             my $d = parse_hostpci($conf->{$key});
5531             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5532
5533             foreach my $pci (@{$d->{pciid}}) {
5534                 my $pciid = $pci->{id};
5535                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5536             }
5537         }
5538
5539         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5540     };
5541     warn $@ if $@; # avoid errors - just warn
5542 }
5543
5544 # call only in locked context
5545 sub _do_vm_stop {
5546     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5547
5548     my $pid = check_running($vmid, $nocheck);
5549     return if !$pid;
5550
5551     my $conf;
5552     if (!$nocheck) {
5553         $conf = PVE::QemuConfig->load_config($vmid);
5554         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5555         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5556             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5557             $timeout = $opts->{down} if $opts->{down};
5558         }
5559         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5560     }
5561
5562     eval {
5563         if ($shutdown) {
5564             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5565                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5566             } else {
5567                 mon_cmd($vmid, "system_powerdown");
5568             }
5569         } else {
5570             mon_cmd($vmid, "quit");
5571         }
5572     };
5573     my $err = $@;
5574
5575     if (!$err) {
5576         $timeout = 60 if !defined($timeout);
5577
5578         my $count = 0;
5579         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5580             $count++;
5581             sleep 1;
5582         }
5583
5584         if ($count >= $timeout) {
5585             if ($force) {
5586                 warn "VM still running - terminating now with SIGTERM\n";
5587                 kill 15, $pid;
5588             } else {
5589                 die "VM quit/powerdown failed - got timeout\n";
5590             }
5591         } else {
5592             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5593             return;
5594         }
5595     } else {
5596        if (!check_running($vmid, $nocheck)) {
5597             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5598             return;
5599        }
5600        if ($force) {
5601             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5602             kill 15, $pid;
5603         } else {
5604             die "VM quit/powerdown failed\n";
5605         }
5606     }
5607
5608     # wait again
5609     $timeout = 10;
5610
5611     my $count = 0;
5612     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5613         $count++;
5614         sleep 1;
5615     }
5616
5617     if ($count >= $timeout) {
5618         warn "VM still running - terminating now with SIGKILL\n";
5619         kill 9, $pid;
5620         sleep 1;
5621     }
5622
5623     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5624 }
5625
5626 # Note: use $nocheck to skip tests if VM configuration file exists.
5627 # We need that when migration VMs to other nodes (files already moved)
5628 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5629 sub vm_stop {
5630     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5631
5632     $force = 1 if !defined($force) && !$shutdown;
5633
5634     if ($migratedfrom){
5635         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5636         kill 15, $pid if $pid;
5637         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5638         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5639         return;
5640     }
5641
5642     PVE::QemuConfig->lock_config($vmid, sub {
5643         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5644    });
5645 }
5646
5647 sub vm_reboot {
5648     my ($vmid, $timeout) = @_;
5649
5650     PVE::QemuConfig->lock_config($vmid, sub {
5651         eval {
5652
5653             # only reboot if running, as qmeventd starts it again on a stop event
5654             return if !check_running($vmid);
5655
5656             create_reboot_request($vmid);
5657
5658             my $storecfg = PVE::Storage::config();
5659             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5660
5661         };
5662         if (my $err = $@) {
5663             # avoid that the next normal shutdown will be confused for a reboot
5664             clear_reboot_request($vmid);
5665             die $err;
5666         }
5667    });
5668 }
5669
5670 # note: if using the statestorage parameter, the caller has to check privileges
5671 sub vm_suspend {
5672     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5673
5674     my $conf;
5675     my $path;
5676     my $storecfg;
5677     my $vmstate;
5678
5679     PVE::QemuConfig->lock_config($vmid, sub {
5680
5681         $conf = PVE::QemuConfig->load_config($vmid);
5682
5683         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5684         PVE::QemuConfig->check_lock($conf)
5685             if !($skiplock || $is_backing_up);
5686
5687         die "cannot suspend to disk during backup\n"
5688             if $is_backing_up && $includestate;
5689
5690         if ($includestate) {
5691             $conf->{lock} = 'suspending';
5692             my $date = strftime("%Y-%m-%d", localtime(time()));
5693             $storecfg = PVE::Storage::config();
5694             if (!$statestorage) {
5695                 $statestorage = find_vmstate_storage($conf, $storecfg);
5696                 # check permissions for the storage
5697                 my $rpcenv = PVE::RPCEnvironment::get();
5698                 if ($rpcenv->{type} ne 'cli') {
5699                     my $authuser = $rpcenv->get_user();
5700                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5701                 }
5702             }
5703
5704
5705             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5706                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5707             $path = PVE::Storage::path($storecfg, $vmstate);
5708             PVE::QemuConfig->write_config($vmid, $conf);
5709         } else {
5710             mon_cmd($vmid, "stop");
5711         }
5712     });
5713
5714     if ($includestate) {
5715         # save vm state
5716         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5717
5718         eval {
5719             set_migration_caps($vmid, 1);
5720             mon_cmd($vmid, "savevm-start", statefile => $path);
5721             for(;;) {
5722                 my $state = mon_cmd($vmid, "query-savevm");
5723                 if (!$state->{status}) {
5724                     die "savevm not active\n";
5725                 } elsif ($state->{status} eq 'active') {
5726                     sleep(1);
5727                     next;
5728                 } elsif ($state->{status} eq 'completed') {
5729                     print "State saved, quitting\n";
5730                     last;
5731                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5732                     die "query-savevm failed with error '$state->{error}'\n"
5733                 } else {
5734                     die "query-savevm returned status '$state->{status}'\n";
5735                 }
5736             }
5737         };
5738         my $err = $@;
5739
5740         PVE::QemuConfig->lock_config($vmid, sub {
5741             $conf = PVE::QemuConfig->load_config($vmid);
5742             if ($err) {
5743                 # cleanup, but leave suspending lock, to indicate something went wrong
5744                 eval {
5745                     mon_cmd($vmid, "savevm-end");
5746                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5747                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5748                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5749                     PVE::QemuConfig->write_config($vmid, $conf);
5750                 };
5751                 warn $@ if $@;
5752                 die $err;
5753             }
5754
5755             die "lock changed unexpectedly\n"
5756                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5757
5758             mon_cmd($vmid, "quit");
5759             $conf->{lock} = 'suspended';
5760             PVE::QemuConfig->write_config($vmid, $conf);
5761         });
5762     }
5763 }
5764
5765 sub vm_resume {
5766     my ($vmid, $skiplock, $nocheck) = @_;
5767
5768     PVE::QemuConfig->lock_config($vmid, sub {
5769         my $res = mon_cmd($vmid, 'query-status');
5770         my $resume_cmd = 'cont';
5771         my $reset = 0;
5772
5773         if ($res->{status}) {
5774             return if $res->{status} eq 'running'; # job done, go home
5775             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5776             $reset = 1 if $res->{status} eq 'shutdown';
5777         }
5778
5779         if (!$nocheck) {
5780
5781             my $conf = PVE::QemuConfig->load_config($vmid);
5782
5783             PVE::QemuConfig->check_lock($conf)
5784                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5785         }
5786
5787         if ($reset) {
5788             # required if a VM shuts down during a backup and we get a resume
5789             # request before the backup finishes for example
5790             mon_cmd($vmid, "system_reset");
5791         }
5792         mon_cmd($vmid, $resume_cmd);
5793     });
5794 }
5795
5796 sub vm_sendkey {
5797     my ($vmid, $skiplock, $key) = @_;
5798
5799     PVE::QemuConfig->lock_config($vmid, sub {
5800
5801         my $conf = PVE::QemuConfig->load_config($vmid);
5802
5803         # there is no qmp command, so we use the human monitor command
5804         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5805         die $res if $res ne '';
5806     });
5807 }
5808
5809 # vzdump restore implementaion
5810
5811 sub tar_archive_read_firstfile {
5812     my $archive = shift;
5813
5814     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5815
5816     # try to detect archive type first
5817     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5818         die "unable to open file '$archive'\n";
5819     my $firstfile = <$fh>;
5820     kill 15, $pid;
5821     close $fh;
5822
5823     die "ERROR: archive contaions no data\n" if !$firstfile;
5824     chomp $firstfile;
5825
5826     return $firstfile;
5827 }
5828
5829 sub tar_restore_cleanup {
5830     my ($storecfg, $statfile) = @_;
5831
5832     print STDERR "starting cleanup\n";
5833
5834     if (my $fd = IO::File->new($statfile, "r")) {
5835         while (defined(my $line = <$fd>)) {
5836             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5837                 my $volid = $2;
5838                 eval {
5839                     if ($volid =~ m|^/|) {
5840                         unlink $volid || die 'unlink failed\n';
5841                     } else {
5842                         PVE::Storage::vdisk_free($storecfg, $volid);
5843                     }
5844                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5845                 };
5846                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5847             } else {
5848                 print STDERR "unable to parse line in statfile - $line";
5849             }
5850         }
5851         $fd->close();
5852     }
5853 }
5854
5855 sub restore_file_archive {
5856     my ($archive, $vmid, $user, $opts) = @_;
5857
5858     return restore_vma_archive($archive, $vmid, $user, $opts)
5859         if $archive eq '-';
5860
5861     my $info = PVE::Storage::archive_info($archive);
5862     my $format = $opts->{format} // $info->{format};
5863     my $comp = $info->{compression};
5864
5865     # try to detect archive format
5866     if ($format eq 'tar') {
5867         return restore_tar_archive($archive, $vmid, $user, $opts);
5868     } else {
5869         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5870     }
5871 }
5872
5873 # hepler to remove disks that will not be used after restore
5874 my $restore_cleanup_oldconf = sub {
5875     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5876
5877     PVE::QemuConfig->foreach_volume($oldconf, sub {
5878         my ($ds, $drive) = @_;
5879
5880         return if drive_is_cdrom($drive, 1);
5881
5882         my $volid = $drive->{file};
5883         return if !$volid || $volid =~ m|^/|;
5884
5885         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5886         return if !$path || !$owner || ($owner != $vmid);
5887
5888         # Note: only delete disk we want to restore
5889         # other volumes will become unused
5890         if ($virtdev_hash->{$ds}) {
5891             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5892             if (my $err = $@) {
5893                 warn $err;
5894             }
5895         }
5896     });
5897
5898     # delete vmstate files, after the restore we have no snapshots anymore
5899     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5900         my $snap = $oldconf->{snapshots}->{$snapname};
5901         if ($snap->{vmstate}) {
5902             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5903             if (my $err = $@) {
5904                 warn $err;
5905             }
5906         }
5907     }
5908 };
5909
5910 # Helper to parse vzdump backup device hints
5911 #
5912 # $rpcenv: Environment, used to ckeck storage permissions
5913 # $user: User ID, to check storage permissions
5914 # $storecfg: Storage configuration
5915 # $fh: the file handle for reading the configuration
5916 # $devinfo: should contain device sizes for all backu-up'ed devices
5917 # $options: backup options (pool, default storage)
5918 #
5919 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5920 my $parse_backup_hints = sub {
5921     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5922
5923     my $virtdev_hash = {};
5924
5925     while (defined(my $line = <$fh>)) {
5926         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5927             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5928             die "archive does not contain data for drive '$virtdev'\n"
5929                 if !$devinfo->{$devname};
5930
5931             if (defined($options->{storage})) {
5932                 $storeid = $options->{storage} || 'local';
5933             } elsif (!$storeid) {
5934                 $storeid = 'local';
5935             }
5936             $format = 'raw' if !$format;
5937             $devinfo->{$devname}->{devname} = $devname;
5938             $devinfo->{$devname}->{virtdev} = $virtdev;
5939             $devinfo->{$devname}->{format} = $format;
5940             $devinfo->{$devname}->{storeid} = $storeid;
5941
5942             # check permission on storage
5943             my $pool = $options->{pool}; # todo: do we need that?
5944             if ($user ne 'root@pam') {
5945                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5946             }
5947
5948             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5949         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5950             my $virtdev = $1;
5951             my $drive = parse_drive($virtdev, $2);
5952             if (drive_is_cloudinit($drive)) {
5953                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5954                 $storeid = $options->{storage} if defined ($options->{storage});
5955                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5956                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5957
5958                 $virtdev_hash->{$virtdev} = {
5959                     format => $format,
5960                     storeid => $storeid,
5961                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5962                     is_cloudinit => 1,
5963                 };
5964             }
5965         }
5966     }
5967
5968     return $virtdev_hash;
5969 };
5970
5971 # Helper to allocate and activate all volumes required for a restore
5972 #
5973 # $storecfg: Storage configuration
5974 # $virtdev_hash: as returned by parse_backup_hints()
5975 #
5976 # Returns: { $virtdev => $volid }
5977 my $restore_allocate_devices = sub {
5978     my ($storecfg, $virtdev_hash, $vmid) = @_;
5979
5980     my $map = {};
5981     foreach my $virtdev (sort keys %$virtdev_hash) {
5982         my $d = $virtdev_hash->{$virtdev};
5983         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5984         my $storeid = $d->{storeid};
5985         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5986
5987         # test if requested format is supported
5988         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5989         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5990         $d->{format} = $defFormat if !$supported;
5991
5992         my $name;
5993         if ($d->{is_cloudinit}) {
5994             $name = "vm-$vmid-cloudinit";
5995             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5996             if ($scfg->{path}) {
5997                 $name .= ".$d->{format}";
5998             }
5999         }
6000
6001         my $volid = PVE::Storage::vdisk_alloc(
6002             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6003
6004         print STDERR "new volume ID is '$volid'\n";
6005         $d->{volid} = $volid;
6006
6007         PVE::Storage::activate_volumes($storecfg, [$volid]);
6008
6009         $map->{$virtdev} = $volid;
6010     }
6011
6012     return $map;
6013 };
6014
6015 sub restore_update_config_line {
6016     my ($cookie, $map, $line, $unique) = @_;
6017
6018     return '' if $line =~ m/^\#qmdump\#/;
6019     return '' if $line =~ m/^\#vzdump\#/;
6020     return '' if $line =~ m/^lock:/;
6021     return '' if $line =~ m/^unused\d+:/;
6022     return '' if $line =~ m/^parent:/;
6023
6024     my $res = '';
6025
6026     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6027     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6028         # try to convert old 1.X settings
6029         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6030         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6031             my ($model, $macaddr) = split(/\=/, $devconfig);
6032             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6033             my $net = {
6034                 model => $model,
6035                 bridge => "vmbr$ind",
6036                 macaddr => $macaddr,
6037             };
6038             my $netstr = print_net($net);
6039
6040             $res .= "net$cookie->{netcount}: $netstr\n";
6041             $cookie->{netcount}++;
6042         }
6043     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6044         my ($id, $netstr) = ($1, $2);
6045         my $net = parse_net($netstr);
6046         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6047         $netstr = print_net($net);
6048         $res .= "$id: $netstr\n";
6049     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6050         my $virtdev = $1;
6051         my $value = $3;
6052         my $di = parse_drive($virtdev, $value);
6053         if (defined($di->{backup}) && !$di->{backup}) {
6054             $res .= "#$line";
6055         } elsif ($map->{$virtdev}) {
6056             delete $di->{format}; # format can change on restore
6057             $di->{file} = $map->{$virtdev};
6058             $value = print_drive($di);
6059             $res .= "$virtdev: $value\n";
6060         } else {
6061             $res .= $line;
6062         }
6063     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6064         my $vmgenid = $1;
6065         if ($vmgenid ne '0') {
6066             # always generate a new vmgenid if there was a valid one setup
6067             $vmgenid = generate_uuid();
6068         }
6069         $res .= "vmgenid: $vmgenid\n";
6070     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6071         my ($uuid, $uuid_str);
6072         UUID::generate($uuid);
6073         UUID::unparse($uuid, $uuid_str);
6074         my $smbios1 = parse_smbios1($2);
6075         $smbios1->{uuid} = $uuid_str;
6076         $res .= $1.print_smbios1($smbios1)."\n";
6077     } else {
6078         $res .= $line;
6079     }
6080
6081     return $res;
6082 }
6083
6084 my $restore_deactivate_volumes = sub {
6085     my ($storecfg, $devinfo) = @_;
6086
6087     my $vollist = [];
6088     foreach my $devname (keys %$devinfo) {
6089         my $volid = $devinfo->{$devname}->{volid};
6090         push @$vollist, $volid if $volid;
6091     }
6092
6093     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6094 };
6095
6096 my $restore_destroy_volumes = sub {
6097     my ($storecfg, $devinfo) = @_;
6098
6099     foreach my $devname (keys %$devinfo) {
6100         my $volid = $devinfo->{$devname}->{volid};
6101         next if !$volid;
6102         eval {
6103             if ($volid =~ m|^/|) {
6104                 unlink $volid || die 'unlink failed\n';
6105             } else {
6106                 PVE::Storage::vdisk_free($storecfg, $volid);
6107             }
6108             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6109         };
6110         print STDERR "unable to cleanup '$volid' - $@" if $@;
6111     }
6112 };
6113
6114 sub scan_volids {
6115     my ($cfg, $vmid) = @_;
6116
6117     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6118
6119     my $volid_hash = {};
6120     foreach my $storeid (keys %$info) {
6121         foreach my $item (@{$info->{$storeid}}) {
6122             next if !($item->{volid} && $item->{size});
6123             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6124             $volid_hash->{$item->{volid}} = $item;
6125         }
6126     }
6127
6128     return $volid_hash;
6129 }
6130
6131 sub update_disk_config {
6132     my ($vmid, $conf, $volid_hash) = @_;
6133
6134     my $changes;
6135     my $prefix = "VM $vmid";
6136
6137     # used and unused disks
6138     my $referenced = {};
6139
6140     # Note: it is allowed to define multiple storages with same path (alias), so
6141     # we need to check both 'volid' and real 'path' (two different volid can point
6142     # to the same path).
6143
6144     my $referencedpath = {};
6145
6146     # update size info
6147     PVE::QemuConfig->foreach_volume($conf, sub {
6148         my ($opt, $drive) = @_;
6149
6150         my $volid = $drive->{file};
6151         return if !$volid;
6152         my $volume = $volid_hash->{$volid};
6153
6154         # mark volid as "in-use" for next step
6155         $referenced->{$volid} = 1;
6156         if ($volume && (my $path = $volume->{path})) {
6157             $referencedpath->{$path} = 1;
6158         }
6159
6160         return if drive_is_cdrom($drive);
6161         return if !$volume;
6162
6163         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6164         if (defined($updated)) {
6165             $changes = 1;
6166             $conf->{$opt} = print_drive($updated);
6167             print "$prefix ($opt): $msg\n";
6168         }
6169     });
6170
6171     # remove 'unusedX' entry if volume is used
6172     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6173         my ($opt, $drive) = @_;
6174
6175         my $volid = $drive->{file};
6176         return if !$volid;
6177
6178         my $path;
6179         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6180         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6181             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6182             $changes = 1;
6183             delete $conf->{$opt};
6184         }
6185
6186         $referenced->{$volid} = 1;
6187         $referencedpath->{$path} = 1 if $path;
6188     });
6189
6190     foreach my $volid (sort keys %$volid_hash) {
6191         next if $volid =~ m/vm-$vmid-state-/;
6192         next if $referenced->{$volid};
6193         my $path = $volid_hash->{$volid}->{path};
6194         next if !$path; # just to be sure
6195         next if $referencedpath->{$path};
6196         $changes = 1;
6197         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6198         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6199         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6200     }
6201
6202     return $changes;
6203 }
6204
6205 sub rescan {
6206     my ($vmid, $nolock, $dryrun) = @_;
6207
6208     my $cfg = PVE::Storage::config();
6209
6210     print "rescan volumes...\n";
6211     my $volid_hash = scan_volids($cfg, $vmid);
6212
6213     my $updatefn =  sub {
6214         my ($vmid) = @_;
6215
6216         my $conf = PVE::QemuConfig->load_config($vmid);
6217
6218         PVE::QemuConfig->check_lock($conf);
6219
6220         my $vm_volids = {};
6221         foreach my $volid (keys %$volid_hash) {
6222             my $info = $volid_hash->{$volid};
6223             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6224         }
6225
6226         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6227
6228         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6229     };
6230
6231     if (defined($vmid)) {
6232         if ($nolock) {
6233             &$updatefn($vmid);
6234         } else {
6235             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6236         }
6237     } else {
6238         my $vmlist = config_list();
6239         foreach my $vmid (keys %$vmlist) {
6240             if ($nolock) {
6241                 &$updatefn($vmid);
6242             } else {
6243                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6244             }
6245         }
6246     }
6247 }
6248
6249 sub restore_proxmox_backup_archive {
6250     my ($archive, $vmid, $user, $options) = @_;
6251
6252     my $storecfg = PVE::Storage::config();
6253
6254     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6255     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6256
6257     my $fingerprint = $scfg->{fingerprint};
6258     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6259
6260     my $repo = PVE::PBSClient::get_repository($scfg);
6261
6262     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6263     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6264     local $ENV{PBS_PASSWORD} = $password;
6265     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6266
6267     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6268         PVE::Storage::parse_volname($storecfg, $archive);
6269
6270     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6271
6272     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6273
6274     my $tmpdir = "/var/tmp/vzdumptmp$$";
6275     rmtree $tmpdir;
6276     mkpath $tmpdir;
6277
6278     my $conffile = PVE::QemuConfig->config_file($vmid);
6279      # disable interrupts (always do cleanups)
6280     local $SIG{INT} =
6281         local $SIG{TERM} =
6282         local $SIG{QUIT} =
6283         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6284
6285     # Note: $oldconf is undef if VM does not exists
6286     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6287     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6288     my $new_conf_raw = '';
6289
6290     my $rpcenv = PVE::RPCEnvironment::get();
6291     my $devinfo = {};
6292
6293     eval {
6294         # enable interrupts
6295         local $SIG{INT} =
6296             local $SIG{TERM} =
6297             local $SIG{QUIT} =
6298             local $SIG{HUP} =
6299             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6300
6301         my $cfgfn = "$tmpdir/qemu-server.conf";
6302         my $firewall_config_fn = "$tmpdir/fw.conf";
6303         my $index_fn = "$tmpdir/index.json";
6304
6305         my $cmd = "restore";
6306
6307         my $param = [$pbs_backup_name, "index.json", $index_fn];
6308         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6309         my $index = PVE::Tools::file_get_contents($index_fn);
6310         $index = decode_json($index);
6311
6312         # print Dumper($index);
6313         foreach my $info (@{$index->{files}}) {
6314             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6315                 my $devname = $1;
6316                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6317                     $devinfo->{$devname}->{size} = $1;
6318                 } else {
6319                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6320                 }
6321             }
6322         }
6323
6324         my $is_qemu_server_backup = scalar(
6325             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6326         );
6327         if (!$is_qemu_server_backup) {
6328             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6329         }
6330         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6331
6332         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6333         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6334
6335         if ($has_firewall_config) {
6336             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6337             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6338
6339             my $pve_firewall_dir = '/etc/pve/firewall';
6340             mkdir $pve_firewall_dir; # make sure the dir exists
6341             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6342         }
6343
6344         my $fh = IO::File->new($cfgfn, "r") ||
6345             die "unable to read qemu-server.conf - $!\n";
6346
6347         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6348
6349         # fixme: rate limit?
6350
6351         # create empty/temp config
6352         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6353
6354         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6355
6356         # allocate volumes
6357         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6358
6359         if (!$options->{live}) {
6360             foreach my $virtdev (sort keys %$virtdev_hash) {
6361                 my $d = $virtdev_hash->{$virtdev};
6362                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6363
6364                 my $volid = $d->{volid};
6365
6366                 my $path = PVE::Storage::path($storecfg, $volid);
6367
6368                 my $pbs_restore_cmd = [
6369                     '/usr/bin/pbs-restore',
6370                     '--repository', $repo,
6371                     $pbs_backup_name,
6372                     "$d->{devname}.img.fidx",
6373                     $path,
6374                     '--verbose',
6375                     ];
6376
6377                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6378                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6379
6380                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6381                     push @$pbs_restore_cmd, '--skip-zero';
6382                 }
6383
6384                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6385                 print "restore proxmox backup image: $dbg_cmdstring\n";
6386                 run_command($pbs_restore_cmd);
6387             }
6388         }
6389
6390         $fh->seek(0, 0) || die "seek failed - $!\n";
6391
6392         my $cookie = { netcount => 0 };
6393         while (defined(my $line = <$fh>)) {
6394             $new_conf_raw .= restore_update_config_line(
6395                 $cookie,
6396                 $map,
6397                 $line,
6398                 $options->{unique},
6399             );
6400         }
6401
6402         $fh->close();
6403     };
6404     my $err = $@;
6405
6406     if ($err || !$options->{live}) {
6407         $restore_deactivate_volumes->($storecfg, $devinfo);
6408     }
6409
6410     rmtree $tmpdir;
6411
6412     if ($err) {
6413         $restore_destroy_volumes->($storecfg, $devinfo);
6414         die $err;
6415     }
6416
6417     if ($options->{live}) {
6418         # keep lock during live-restore
6419         $new_conf_raw .= "\nlock: create";
6420     }
6421
6422     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6423
6424     PVE::Cluster::cfs_update(); # make sure we read new file
6425
6426     eval { rescan($vmid, 1); };
6427     warn $@ if $@;
6428
6429     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6430
6431     if ($options->{live}) {
6432         # enable interrupts
6433         local $SIG{INT} =
6434             local $SIG{TERM} =
6435             local $SIG{QUIT} =
6436             local $SIG{HUP} =
6437             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6438
6439         my $conf = PVE::QemuConfig->load_config($vmid);
6440         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6441
6442         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6443
6444         PVE::QemuConfig->remove_lock($vmid, "create");
6445     }
6446 }
6447
6448 sub pbs_live_restore {
6449     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6450
6451     print "starting VM for live-restore\n";
6452     print "repository: '$repo', snapshot: '$snap'\n";
6453
6454     my $pbs_backing = {};
6455     for my $ds (keys %$restored_disks) {
6456         $ds =~ m/^drive-(.*)$/;
6457         my $confname = $1;
6458         $pbs_backing->{$confname} = {
6459             repository => $repo,
6460             snapshot => $snap,
6461             archive => "$ds.img.fidx",
6462         };
6463         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6464
6465         my $drive = parse_drive($confname, $conf->{$confname});
6466         print "restoring '$ds' to '$drive->{file}'\n";
6467     }
6468
6469     my $drives_streamed = 0;
6470     eval {
6471         # make sure HA doesn't interrupt our restore by stopping the VM
6472         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6473             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6474         }
6475
6476         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6477         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6478         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6479
6480         my $qmeventd_fd = register_qmeventd_handle($vmid);
6481
6482         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6483         # this will effectively collapse the backing image chain consisting of
6484         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6485         # removes itself once all backing images vanish with 'auto-remove=on')
6486         my $jobs = {};
6487         for my $ds (sort keys %$restored_disks) {
6488             my $job_id = "restore-$ds";
6489             mon_cmd($vmid, 'block-stream',
6490                 'job-id' => $job_id,
6491                 device => "$ds",
6492             );
6493             $jobs->{$job_id} = {};
6494         }
6495
6496         mon_cmd($vmid, 'cont');
6497         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6498
6499         print "restore-drive jobs finished successfully, removing all tracking block devices"
6500             ." to disconnect from Proxmox Backup Server\n";
6501
6502         for my $ds (sort keys %$restored_disks) {
6503             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6504         }
6505
6506         close($qmeventd_fd);
6507     };
6508
6509     my $err = $@;
6510
6511     if ($err) {
6512         warn "An error occured during live-restore: $err\n";
6513         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6514         die "live-restore failed\n";
6515     }
6516 }
6517
6518 sub restore_vma_archive {
6519     my ($archive, $vmid, $user, $opts, $comp) = @_;
6520
6521     my $readfrom = $archive;
6522
6523     my $cfg = PVE::Storage::config();
6524     my $commands = [];
6525     my $bwlimit = $opts->{bwlimit};
6526
6527     my $dbg_cmdstring = '';
6528     my $add_pipe = sub {
6529         my ($cmd) = @_;
6530         push @$commands, $cmd;
6531         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6532         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6533         $readfrom = '-';
6534     };
6535
6536     my $input = undef;
6537     if ($archive eq '-') {
6538         $input = '<&STDIN';
6539     } else {
6540         # If we use a backup from a PVE defined storage we also consider that
6541         # storage's rate limit:
6542         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6543         if (defined($volid)) {
6544             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6545             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6546             if ($readlimit) {
6547                 print STDERR "applying read rate limit: $readlimit\n";
6548                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6549                 $add_pipe->($cstream);
6550             }
6551         }
6552     }
6553
6554     if ($comp) {
6555         my $info = PVE::Storage::decompressor_info('vma', $comp);
6556         my $cmd = $info->{decompressor};
6557         push @$cmd, $readfrom;
6558         $add_pipe->($cmd);
6559     }
6560
6561     my $tmpdir = "/var/tmp/vzdumptmp$$";
6562     rmtree $tmpdir;
6563
6564     # disable interrupts (always do cleanups)
6565     local $SIG{INT} =
6566         local $SIG{TERM} =
6567         local $SIG{QUIT} =
6568         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6569
6570     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6571     POSIX::mkfifo($mapfifo, 0600);
6572     my $fifofh;
6573     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6574
6575     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6576
6577     my $oldtimeout;
6578     my $timeout = 5;
6579
6580     my $devinfo = {};
6581
6582     my $rpcenv = PVE::RPCEnvironment::get();
6583
6584     my $conffile = PVE::QemuConfig->config_file($vmid);
6585
6586     # Note: $oldconf is undef if VM does not exist
6587     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6588     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6589     my $new_conf_raw = '';
6590
6591     my %storage_limits;
6592
6593     my $print_devmap = sub {
6594         my $cfgfn = "$tmpdir/qemu-server.conf";
6595
6596         # we can read the config - that is already extracted
6597         my $fh = IO::File->new($cfgfn, "r") ||
6598             die "unable to read qemu-server.conf - $!\n";
6599
6600         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6601         if (-f $fwcfgfn) {
6602             my $pve_firewall_dir = '/etc/pve/firewall';
6603             mkdir $pve_firewall_dir; # make sure the dir exists
6604             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6605         }
6606
6607         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6608
6609         foreach my $info (values %{$virtdev_hash}) {
6610             my $storeid = $info->{storeid};
6611             next if defined($storage_limits{$storeid});
6612
6613             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6614             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6615             $storage_limits{$storeid} = $limit * 1024;
6616         }
6617
6618         foreach my $devname (keys %$devinfo) {
6619             die "found no device mapping information for device '$devname'\n"
6620                 if !$devinfo->{$devname}->{virtdev};
6621         }
6622
6623         # create empty/temp config
6624         if ($oldconf) {
6625             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6626             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6627         }
6628
6629         # allocate volumes
6630         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6631
6632         # print restore information to $fifofh
6633         foreach my $virtdev (sort keys %$virtdev_hash) {
6634             my $d = $virtdev_hash->{$virtdev};
6635             next if $d->{is_cloudinit}; # no need to restore cloudinit
6636
6637             my $storeid = $d->{storeid};
6638             my $volid = $d->{volid};
6639
6640             my $map_opts = '';
6641             if (my $limit = $storage_limits{$storeid}) {
6642                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6643             }
6644
6645             my $write_zeros = 1;
6646             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6647                 $write_zeros = 0;
6648             }
6649
6650             my $path = PVE::Storage::path($cfg, $volid);
6651
6652             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6653
6654             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6655         }
6656
6657         $fh->seek(0, 0) || die "seek failed - $!\n";
6658
6659         my $cookie = { netcount => 0 };
6660         while (defined(my $line = <$fh>)) {
6661             $new_conf_raw .= restore_update_config_line(
6662                 $cookie,
6663                 $map,
6664                 $line,
6665                 $opts->{unique},
6666             );
6667         }
6668
6669         $fh->close();
6670     };
6671
6672     eval {
6673         # enable interrupts
6674         local $SIG{INT} =
6675             local $SIG{TERM} =
6676             local $SIG{QUIT} =
6677             local $SIG{HUP} =
6678             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6679         local $SIG{ALRM} = sub { die "got timeout\n"; };
6680
6681         $oldtimeout = alarm($timeout);
6682
6683         my $parser = sub {
6684             my $line = shift;
6685
6686             print "$line\n";
6687
6688             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6689                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6690                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6691             } elsif ($line =~ m/^CTIME: /) {
6692                 # we correctly received the vma config, so we can disable
6693                 # the timeout now for disk allocation (set to 10 minutes, so
6694                 # that we always timeout if something goes wrong)
6695                 alarm(600);
6696                 &$print_devmap();
6697                 print $fifofh "done\n";
6698                 my $tmp = $oldtimeout || 0;
6699                 $oldtimeout = undef;
6700                 alarm($tmp);
6701                 close($fifofh);
6702                 $fifofh = undef;
6703             }
6704         };
6705
6706         print "restore vma archive: $dbg_cmdstring\n";
6707         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6708     };
6709     my $err = $@;
6710
6711     alarm($oldtimeout) if $oldtimeout;
6712
6713     $restore_deactivate_volumes->($cfg, $devinfo);
6714
6715     close($fifofh) if $fifofh;
6716     unlink $mapfifo;
6717     rmtree $tmpdir;
6718
6719     if ($err) {
6720         $restore_destroy_volumes->($cfg, $devinfo);
6721         die $err;
6722     }
6723
6724     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6725
6726     PVE::Cluster::cfs_update(); # make sure we read new file
6727
6728     eval { rescan($vmid, 1); };
6729     warn $@ if $@;
6730
6731     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6732 }
6733
6734 sub restore_tar_archive {
6735     my ($archive, $vmid, $user, $opts) = @_;
6736
6737     if ($archive ne '-') {
6738         my $firstfile = tar_archive_read_firstfile($archive);
6739         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6740             if $firstfile ne 'qemu-server.conf';
6741     }
6742
6743     my $storecfg = PVE::Storage::config();
6744
6745     # avoid zombie disks when restoring over an existing VM -> cleanup first
6746     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6747     # skiplock=1 because qmrestore has set the 'create' lock itself already
6748     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6749     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6750
6751     my $tocmd = "/usr/lib/qemu-server/qmextract";
6752
6753     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6754     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6755     $tocmd .= ' --prealloc' if $opts->{prealloc};
6756     $tocmd .= ' --info' if $opts->{info};
6757
6758     # tar option "xf" does not autodetect compression when read from STDIN,
6759     # so we pipe to zcat
6760     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6761         PVE::Tools::shellquote("--to-command=$tocmd");
6762
6763     my $tmpdir = "/var/tmp/vzdumptmp$$";
6764     mkpath $tmpdir;
6765
6766     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6767     local $ENV{VZDUMP_VMID} = $vmid;
6768     local $ENV{VZDUMP_USER} = $user;
6769
6770     my $conffile = PVE::QemuConfig->config_file($vmid);
6771     my $new_conf_raw = '';
6772
6773     # disable interrupts (always do cleanups)
6774     local $SIG{INT} =
6775         local $SIG{TERM} =
6776         local $SIG{QUIT} =
6777         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6778
6779     eval {
6780         # enable interrupts
6781         local $SIG{INT} =
6782             local $SIG{TERM} =
6783             local $SIG{QUIT} =
6784             local $SIG{HUP} =
6785             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6786
6787         if ($archive eq '-') {
6788             print "extracting archive from STDIN\n";
6789             run_command($cmd, input => "<&STDIN");
6790         } else {
6791             print "extracting archive '$archive'\n";
6792             run_command($cmd);
6793         }
6794
6795         return if $opts->{info};
6796
6797         # read new mapping
6798         my $map = {};
6799         my $statfile = "$tmpdir/qmrestore.stat";
6800         if (my $fd = IO::File->new($statfile, "r")) {
6801             while (defined (my $line = <$fd>)) {
6802                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6803                     $map->{$1} = $2 if $1;
6804                 } else {
6805                     print STDERR "unable to parse line in statfile - $line\n";
6806                 }
6807             }
6808             $fd->close();
6809         }
6810
6811         my $confsrc = "$tmpdir/qemu-server.conf";
6812
6813         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6814
6815         my $cookie = { netcount => 0 };
6816         while (defined (my $line = <$srcfd>)) {
6817             $new_conf_raw .= restore_update_config_line(
6818                 $cookie,
6819                 $map,
6820                 $line,
6821                 $opts->{unique},
6822             );
6823         }
6824
6825         $srcfd->close();
6826     };
6827     if (my $err = $@) {
6828         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6829         die $err;
6830     }
6831
6832     rmtree $tmpdir;
6833
6834     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6835
6836     PVE::Cluster::cfs_update(); # make sure we read new file
6837
6838     eval { rescan($vmid, 1); };
6839     warn $@ if $@;
6840 };
6841
6842 sub foreach_storage_used_by_vm {
6843     my ($conf, $func) = @_;
6844
6845     my $sidhash = {};
6846
6847     PVE::QemuConfig->foreach_volume($conf, sub {
6848         my ($ds, $drive) = @_;
6849         return if drive_is_cdrom($drive);
6850
6851         my $volid = $drive->{file};
6852
6853         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6854         $sidhash->{$sid} = $sid if $sid;
6855     });
6856
6857     foreach my $sid (sort keys %$sidhash) {
6858         &$func($sid);
6859     }
6860 }
6861
6862 my $qemu_snap_storage = {
6863     rbd => 1,
6864 };
6865 sub do_snapshots_with_qemu {
6866     my ($storecfg, $volid) = @_;
6867
6868     my $storage_name = PVE::Storage::parse_volume_id($volid);
6869     my $scfg = $storecfg->{ids}->{$storage_name};
6870     die "could not find storage '$storage_name'\n" if !defined($scfg);
6871
6872     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6873         return 1;
6874     }
6875
6876     if ($volid =~ m/\.(qcow2|qed)$/){
6877         return 1;
6878     }
6879
6880     return;
6881 }
6882
6883 sub qga_check_running {
6884     my ($vmid, $nowarn) = @_;
6885
6886     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6887     if ($@) {
6888         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6889         return 0;
6890     }
6891     return 1;
6892 }
6893
6894 sub template_create {
6895     my ($vmid, $conf, $disk) = @_;
6896
6897     my $storecfg = PVE::Storage::config();
6898
6899     PVE::QemuConfig->foreach_volume($conf, sub {
6900         my ($ds, $drive) = @_;
6901
6902         return if drive_is_cdrom($drive);
6903         return if $disk && $ds ne $disk;
6904
6905         my $volid = $drive->{file};
6906         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6907
6908         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6909         $drive->{file} = $voliddst;
6910         $conf->{$ds} = print_drive($drive);
6911         PVE::QemuConfig->write_config($vmid, $conf);
6912     });
6913 }
6914
6915 sub convert_iscsi_path {
6916     my ($path) = @_;
6917
6918     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6919         my $portal = $1;
6920         my $target = $2;
6921         my $lun = $3;
6922
6923         my $initiator_name = get_initiator_name();
6924
6925         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6926                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6927     }
6928
6929     die "cannot convert iscsi path '$path', unkown format\n";
6930 }
6931
6932 sub qemu_img_convert {
6933     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6934
6935     my $storecfg = PVE::Storage::config();
6936     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6937     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6938
6939     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6940
6941     my $cachemode;
6942     my $src_path;
6943     my $src_is_iscsi = 0;
6944     my $src_format;
6945
6946     if ($src_storeid) {
6947         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6948         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6949         $src_format = qemu_img_format($src_scfg, $src_volname);
6950         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6951         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6952         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6953     } elsif (-f $src_volid) {
6954         $src_path = $src_volid;
6955         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6956             $src_format = $1;
6957         }
6958     }
6959
6960     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6961
6962     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6963     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6964     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6965     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6966
6967     my $cmd = [];
6968     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6969     push @$cmd, '-l', "snapshot.name=$snapname"
6970         if $snapname && $src_format && $src_format eq "qcow2";
6971     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6972     push @$cmd, '-T', $cachemode if defined($cachemode);
6973
6974     if ($src_is_iscsi) {
6975         push @$cmd, '--image-opts';
6976         $src_path = convert_iscsi_path($src_path);
6977     } elsif ($src_format) {
6978         push @$cmd, '-f', $src_format;
6979     }
6980
6981     if ($dst_is_iscsi) {
6982         push @$cmd, '--target-image-opts';
6983         $dst_path = convert_iscsi_path($dst_path);
6984     } else {
6985         push @$cmd, '-O', $dst_format;
6986     }
6987
6988     push @$cmd, $src_path;
6989
6990     if (!$dst_is_iscsi && $is_zero_initialized) {
6991         push @$cmd, "zeroinit:$dst_path";
6992     } else {
6993         push @$cmd, $dst_path;
6994     }
6995
6996     my $parser = sub {
6997         my $line = shift;
6998         if($line =~ m/\((\S+)\/100\%\)/){
6999             my $percent = $1;
7000             my $transferred = int($size * $percent / 100);
7001             my $total_h = render_bytes($size, 1);
7002             my $transferred_h = render_bytes($transferred, 1);
7003
7004             print "transferred $transferred_h of $total_h ($percent%)\n";
7005         }
7006
7007     };
7008
7009     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7010     my $err = $@;
7011     die "copy failed: $err" if $err;
7012 }
7013
7014 sub qemu_img_format {
7015     my ($scfg, $volname) = @_;
7016
7017     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7018         return $1;
7019     } else {
7020         return "raw";
7021     }
7022 }
7023
7024 sub qemu_drive_mirror {
7025     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7026
7027     $jobs = {} if !$jobs;
7028
7029     my $qemu_target;
7030     my $format;
7031     $jobs->{"drive-$drive"} = {};
7032
7033     if ($dst_volid =~ /^nbd:/) {
7034         $qemu_target = $dst_volid;
7035         $format = "nbd";
7036     } else {
7037         my $storecfg = PVE::Storage::config();
7038         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7039
7040         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7041
7042         $format = qemu_img_format($dst_scfg, $dst_volname);
7043
7044         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7045
7046         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7047     }
7048
7049     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7050     $opts->{format} = $format if $format;
7051
7052     if (defined($src_bitmap)) {
7053         $opts->{sync} = 'incremental';
7054         $opts->{bitmap} = $src_bitmap;
7055         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7056     }
7057
7058     if (defined($bwlimit)) {
7059         $opts->{speed} = $bwlimit * 1024;
7060         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7061     } else {
7062         print "drive mirror is starting for drive-$drive\n";
7063     }
7064
7065     # if a job already runs for this device we get an error, catch it for cleanup
7066     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7067     if (my $err = $@) {
7068         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7069         warn "$@\n" if $@;
7070         die "mirroring error: $err\n";
7071     }
7072
7073     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7074 }
7075
7076 # $completion can be either
7077 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7078 # 'cancel': wait until all jobs are ready, block-job-cancel them
7079 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7080 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7081 sub qemu_drive_mirror_monitor {
7082     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7083
7084     $completion //= 'complete';
7085     $op //= "mirror";
7086
7087     eval {
7088         my $err_complete = 0;
7089
7090         my $starttime = time ();
7091         while (1) {
7092             die "block job ('$op') timed out\n" if $err_complete > 300;
7093
7094             my $stats = mon_cmd($vmid, "query-block-jobs");
7095             my $ctime = time();
7096
7097             my $running_jobs = {};
7098             for my $stat (@$stats) {
7099                 next if $stat->{type} ne $op;
7100                 $running_jobs->{$stat->{device}} = $stat;
7101             }
7102
7103             my $readycounter = 0;
7104
7105             for my $job_id (sort keys %$jobs) {
7106                 my $job = $running_jobs->{$job_id};
7107
7108                 my $vanished = !defined($job);
7109                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7110                 if($complete || ($vanished && $completion eq 'auto')) {
7111                     print "$job_id: $op-job finished\n";
7112                     delete $jobs->{$job_id};
7113                     next;
7114                 }
7115
7116                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7117
7118                 my $busy = $job->{busy};
7119                 my $ready = $job->{ready};
7120                 if (my $total = $job->{len}) {
7121                     my $transferred = $job->{offset} || 0;
7122                     my $remaining = $total - $transferred;
7123                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7124
7125                     my $duration = $ctime - $starttime;
7126                     my $total_h = render_bytes($total, 1);
7127                     my $transferred_h = render_bytes($transferred, 1);
7128
7129                     my $status = sprintf(
7130                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7131                         render_duration($duration),
7132                     );
7133
7134                     if ($ready) {
7135                         if ($busy) {
7136                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7137                         } else {
7138                             $status .= ", ready";
7139                         }
7140                     }
7141                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7142                     $jobs->{$job_id}->{ready} = $ready;
7143                 }
7144
7145                 $readycounter++ if $job->{ready};
7146             }
7147
7148             last if scalar(keys %$jobs) == 0;
7149
7150             if ($readycounter == scalar(keys %$jobs)) {
7151                 print "all '$op' jobs are ready\n";
7152
7153                 # do the complete later (or has already been done)
7154                 last if $completion eq 'skip' || $completion eq 'auto';
7155
7156                 if ($vmiddst && $vmiddst != $vmid) {
7157                     my $agent_running = $qga && qga_check_running($vmid);
7158                     if ($agent_running) {
7159                         print "freeze filesystem\n";
7160                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7161                     } else {
7162                         print "suspend vm\n";
7163                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7164                     }
7165
7166                     # if we clone a disk for a new target vm, we don't switch the disk
7167                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7168
7169                     if ($agent_running) {
7170                         print "unfreeze filesystem\n";
7171                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7172                     } else {
7173                         print "resume vm\n";
7174                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7175                     }
7176
7177                     last;
7178                 } else {
7179
7180                     for my $job_id (sort keys %$jobs) {
7181                         # try to switch the disk if source and destination are on the same guest
7182                         print "$job_id: Completing block job_id...\n";
7183
7184                         my $op;
7185                         if ($completion eq 'complete') {
7186                             $op = 'block-job-complete';
7187                         } elsif ($completion eq 'cancel') {
7188                             $op = 'block-job-cancel';
7189                         } else {
7190                             die "invalid completion value: $completion\n";
7191                         }
7192                         eval { mon_cmd($vmid, $op, device => $job_id) };
7193                         if ($@ =~ m/cannot be completed/) {
7194                             print "$job_id: block job cannot be completed, trying again.\n";
7195                             $err_complete++;
7196                         }else {
7197                             print "$job_id: Completed successfully.\n";
7198                             $jobs->{$job_id}->{complete} = 1;
7199                         }
7200                     }
7201                 }
7202             }
7203             sleep 1;
7204         }
7205     };
7206     my $err = $@;
7207
7208     if ($err) {
7209         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7210         die "block job ($op) error: $err";
7211     }
7212 }
7213
7214 sub qemu_blockjobs_cancel {
7215     my ($vmid, $jobs) = @_;
7216
7217     foreach my $job (keys %$jobs) {
7218         print "$job: Cancelling block job\n";
7219         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7220         $jobs->{$job}->{cancel} = 1;
7221     }
7222
7223     while (1) {
7224         my $stats = mon_cmd($vmid, "query-block-jobs");
7225
7226         my $running_jobs = {};
7227         foreach my $stat (@$stats) {
7228             $running_jobs->{$stat->{device}} = $stat;
7229         }
7230
7231         foreach my $job (keys %$jobs) {
7232
7233             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7234                 print "$job: Done.\n";
7235                 delete $jobs->{$job};
7236             }
7237         }
7238
7239         last if scalar(keys %$jobs) == 0;
7240
7241         sleep 1;
7242     }
7243 }
7244
7245 sub clone_disk {
7246     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7247         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7248
7249     my $newvolid;
7250
7251     if (!$full) {
7252         print "create linked clone of drive $drivename ($drive->{file})\n";
7253         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7254         push @$newvollist, $newvolid;
7255     } else {
7256
7257         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7258         $storeid = $storage if $storage;
7259
7260         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7261
7262         print "create full clone of drive $drivename ($drive->{file})\n";
7263         my $name = undef;
7264         my $size = undef;
7265         if (drive_is_cloudinit($drive)) {
7266             $name = "vm-$newvmid-cloudinit";
7267             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7268             if ($scfg->{path}) {
7269                 $name .= ".$dst_format";
7270             }
7271             $snapname = undef;
7272             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7273         } elsif ($drivename eq 'efidisk0') {
7274             $size = get_efivars_size($conf);
7275         } else {
7276             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7277         }
7278         $newvolid = PVE::Storage::vdisk_alloc(
7279             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7280         );
7281         push @$newvollist, $newvolid;
7282
7283         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7284
7285         if (drive_is_cloudinit($drive)) {
7286             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7287             # if this is the case, we have to complete any block-jobs still there from
7288             # previous drive-mirrors
7289             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7290                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7291             }
7292             goto no_data_clone;
7293         }
7294
7295         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7296         if (!$running || $snapname) {
7297             # TODO: handle bwlimits
7298             if ($drivename eq 'efidisk0') {
7299                 # the relevant data on the efidisk may be smaller than the source
7300                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7301                 # that is given by the OVMF_VARS.fd
7302                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7303                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7304
7305                 # better for Ceph if block size is not too small, see bug #3324
7306                 my $bs = 1024*1024;
7307
7308                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7309                     "if=$src_path", "of=$dst_path"]);
7310             } else {
7311                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7312             }
7313         } else {
7314
7315             my $kvmver = get_running_qemu_version ($vmid);
7316             if (!min_version($kvmver, 2, 7)) {
7317                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7318                     if $drive->{iothread};
7319             }
7320
7321             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7322                 $completion, $qga, $bwlimit);
7323         }
7324     }
7325
7326 no_data_clone:
7327     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7328
7329     my $disk = $drive;
7330     $disk->{format} = undef;
7331     $disk->{file} = $newvolid;
7332     $disk->{size} = $size if defined($size);
7333
7334     return $disk;
7335 }
7336
7337 sub get_running_qemu_version {
7338     my ($vmid) = @_;
7339     my $res = mon_cmd($vmid, "query-version");
7340     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7341 }
7342
7343 sub qemu_use_old_bios_files {
7344     my ($machine_type) = @_;
7345
7346     return if !$machine_type;
7347
7348     my $use_old_bios_files = undef;
7349
7350     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7351         $machine_type = $1;
7352         $use_old_bios_files = 1;
7353     } else {
7354         my $version = extract_version($machine_type, kvm_user_version());
7355         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7356         # load new efi bios files on migration. So this hack is required to allow
7357         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7358         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7359         $use_old_bios_files = !min_version($version, 2, 4);
7360     }
7361
7362     return ($use_old_bios_files, $machine_type);
7363 }
7364
7365 sub get_efivars_size {
7366     my ($conf) = @_;
7367     my $arch = get_vm_arch($conf);
7368     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7369     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7370     return -s $ovmf_vars;
7371 }
7372
7373 sub update_efidisk_size {
7374     my ($conf) = @_;
7375
7376     return if !defined($conf->{efidisk0});
7377
7378     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7379     $disk->{size} = get_efivars_size($conf);
7380     $conf->{efidisk0} = print_drive($disk);
7381
7382     return;
7383 }
7384
7385 sub create_efidisk($$$$$) {
7386     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7387
7388     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7389     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7390
7391     my $vars_size_b = -s $ovmf_vars;
7392     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7393     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7394     PVE::Storage::activate_volumes($storecfg, [$volid]);
7395
7396     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7397     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7398
7399     return ($volid, $size/1024);
7400 }
7401
7402 sub vm_iothreads_list {
7403     my ($vmid) = @_;
7404
7405     my $res = mon_cmd($vmid, 'query-iothreads');
7406
7407     my $iothreads = {};
7408     foreach my $iothread (@$res) {
7409         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7410     }
7411
7412     return $iothreads;
7413 }
7414
7415 sub scsihw_infos {
7416     my ($conf, $drive) = @_;
7417
7418     my $maxdev = 0;
7419
7420     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7421         $maxdev = 7;
7422     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7423         $maxdev = 1;
7424     } else {
7425         $maxdev = 256;
7426     }
7427
7428     my $controller = int($drive->{index} / $maxdev);
7429     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7430         ? "virtioscsi"
7431         : "scsihw";
7432
7433     return ($maxdev, $controller, $controller_prefix);
7434 }
7435
7436 sub windows_version {
7437     my ($ostype) = @_;
7438
7439     return 0 if !$ostype;
7440
7441     my $winversion = 0;
7442
7443     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7444         $winversion = 5;
7445     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7446         $winversion = 6;
7447     } elsif ($ostype =~ m/^win(\d+)$/) {
7448         $winversion = $1;
7449     }
7450
7451     return $winversion;
7452 }
7453
7454 sub resolve_dst_disk_format {
7455         my ($storecfg, $storeid, $src_volname, $format) = @_;
7456         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7457
7458         if (!$format) {
7459             # if no target format is specified, use the source disk format as hint
7460             if ($src_volname) {
7461                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7462                 $format = qemu_img_format($scfg, $src_volname);
7463             } else {
7464                 return $defFormat;
7465             }
7466         }
7467
7468         # test if requested format is supported - else use default
7469         my $supported = grep { $_ eq $format } @$validFormats;
7470         $format = $defFormat if !$supported;
7471         return $format;
7472 }
7473
7474 # NOTE: if this logic changes, please update docs & possibly gui logic
7475 sub find_vmstate_storage {
7476     my ($conf, $storecfg) = @_;
7477
7478     # first, return storage from conf if set
7479     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7480
7481     my ($target, $shared, $local);
7482
7483     foreach_storage_used_by_vm($conf, sub {
7484         my ($sid) = @_;
7485         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7486         my $dst = $scfg->{shared} ? \$shared : \$local;
7487         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7488     });
7489
7490     # second, use shared storage where VM has at least one disk
7491     # third, use local storage where VM has at least one disk
7492     # fall back to local storage
7493     $target = $shared // $local // 'local';
7494
7495     return $target;
7496 }
7497
7498 sub generate_uuid {
7499     my ($uuid, $uuid_str);
7500     UUID::generate($uuid);
7501     UUID::unparse($uuid, $uuid_str);
7502     return $uuid_str;
7503 }
7504
7505 sub generate_smbios1_uuid {
7506     return "uuid=".generate_uuid();
7507 }
7508
7509 sub nbd_stop {
7510     my ($vmid) = @_;
7511
7512     mon_cmd($vmid, 'nbd-server-stop');
7513 }
7514
7515 sub create_reboot_request {
7516     my ($vmid) = @_;
7517     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7518         or die "failed to create reboot trigger file: $!\n";
7519     close($fh);
7520 }
7521
7522 sub clear_reboot_request {
7523     my ($vmid) = @_;
7524     my $path = "/run/qemu-server/$vmid.reboot";
7525     my $res = 0;
7526
7527     $res = unlink($path);
7528     die "could not remove reboot request for $vmid: $!"
7529         if !$res && $! != POSIX::ENOENT;
7530
7531     return $res;
7532 }
7533
7534 sub bootorder_from_legacy {
7535     my ($conf, $bootcfg) = @_;
7536
7537     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7538     my $bootindex_hash = {};
7539     my $i = 1;
7540     foreach my $o (split(//, $boot)) {
7541         $bootindex_hash->{$o} = $i*100;
7542         $i++;
7543     }
7544
7545     my $bootorder = {};
7546
7547     PVE::QemuConfig->foreach_volume($conf, sub {
7548         my ($ds, $drive) = @_;
7549
7550         if (drive_is_cdrom ($drive, 1)) {
7551             if ($bootindex_hash->{d}) {
7552                 $bootorder->{$ds} = $bootindex_hash->{d};
7553                 $bootindex_hash->{d} += 1;
7554             }
7555         } elsif ($bootindex_hash->{c}) {
7556             $bootorder->{$ds} = $bootindex_hash->{c}
7557                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7558             $bootindex_hash->{c} += 1;
7559         }
7560     });
7561
7562     if ($bootindex_hash->{n}) {
7563         for (my $i = 0; $i < $MAX_NETS; $i++) {
7564             my $netname = "net$i";
7565             next if !$conf->{$netname};
7566             $bootorder->{$netname} = $bootindex_hash->{n};
7567             $bootindex_hash->{n} += 1;
7568         }
7569     }
7570
7571     return $bootorder;
7572 }
7573
7574 # Generate default device list for 'boot: order=' property. Matches legacy
7575 # default boot order, but with explicit device names. This is important, since
7576 # the fallback for when neither 'order' nor the old format is specified relies
7577 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7578 sub get_default_bootdevices {
7579     my ($conf) = @_;
7580
7581     my @ret = ();
7582
7583     # harddisk
7584     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7585     push @ret, $first if $first;
7586
7587     # cdrom
7588     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7589     push @ret, $first if $first;
7590
7591     # network
7592     for (my $i = 0; $i < $MAX_NETS; $i++) {
7593         my $netname = "net$i";
7594         next if !$conf->{$netname};
7595         push @ret, $netname;
7596         last;
7597     }
7598
7599     return \@ret;
7600 }
7601
7602 sub device_bootorder {
7603     my ($conf) = @_;
7604
7605     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7606
7607     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7608
7609     my $bootorder = {};
7610     if (!defined($boot) || $boot->{legacy}) {
7611         $bootorder = bootorder_from_legacy($conf, $boot);
7612     } elsif ($boot->{order}) {
7613         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7614         for my $dev (PVE::Tools::split_list($boot->{order})) {
7615             $bootorder->{$dev} = $i++;
7616         }
7617     }
7618
7619     return $bootorder;
7620 }
7621
7622 sub register_qmeventd_handle {
7623     my ($vmid) = @_;
7624
7625     my $fh;
7626     my $peer = "/var/run/qmeventd.sock";
7627     my $count = 0;
7628
7629     for (;;) {
7630         $count++;
7631         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7632         last if $fh;
7633         if ($! != EINTR && $! != EAGAIN) {
7634             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7635         }
7636         if ($count > 4) {
7637             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7638               . "after $count retries\n";
7639         }
7640         usleep(25000);
7641     }
7642
7643     # send handshake to mark VM as backing up
7644     print $fh to_json({vzdump => {vmid => "$vmid"}});
7645
7646     # return handle to be closed later when inhibit is no longer required
7647     return $fh;
7648 }
7649
7650 # bash completion helper
7651
7652 sub complete_backup_archives {
7653     my ($cmdname, $pname, $cvalue) = @_;
7654
7655     my $cfg = PVE::Storage::config();
7656
7657     my $storeid;
7658
7659     if ($cvalue =~ m/^([^:]+):/) {
7660         $storeid = $1;
7661     }
7662
7663     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7664
7665     my $res = [];
7666     foreach my $id (keys %$data) {
7667         foreach my $item (@{$data->{$id}}) {
7668             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7669             push @$res, $item->{volid} if defined($item->{volid});
7670         }
7671     }
7672
7673     return $res;
7674 }
7675
7676 my $complete_vmid_full = sub {
7677     my ($running) = @_;
7678
7679     my $idlist = vmstatus();
7680
7681     my $res = [];
7682
7683     foreach my $id (keys %$idlist) {
7684         my $d = $idlist->{$id};
7685         if (defined($running)) {
7686             next if $d->{template};
7687             next if $running && $d->{status} ne 'running';
7688             next if !$running && $d->{status} eq 'running';
7689         }
7690         push @$res, $id;
7691
7692     }
7693     return $res;
7694 };
7695
7696 sub complete_vmid {
7697     return &$complete_vmid_full();
7698 }
7699
7700 sub complete_vmid_stopped {
7701     return &$complete_vmid_full(0);
7702 }
7703
7704 sub complete_vmid_running {
7705     return &$complete_vmid_full(1);
7706 }
7707
7708 sub complete_storage {
7709
7710     my $cfg = PVE::Storage::config();
7711     my $ids = $cfg->{ids};
7712
7713     my $res = [];
7714     foreach my $sid (keys %$ids) {
7715         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7716         next if !$ids->{$sid}->{content}->{images};
7717         push @$res, $sid;
7718     }
7719
7720     return $res;
7721 }
7722
7723 sub complete_migration_storage {
7724     my ($cmd, $param, $current_value, $all_args) = @_;
7725
7726     my $targetnode = @$all_args[1];
7727
7728     my $cfg = PVE::Storage::config();
7729     my $ids = $cfg->{ids};
7730
7731     my $res = [];
7732     foreach my $sid (keys %$ids) {
7733         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7734         next if !$ids->{$sid}->{content}->{images};
7735         push @$res, $sid;
7736     }
7737
7738     return $res;
7739 }
7740
7741 sub vm_is_paused {
7742     my ($vmid) = @_;
7743     my $qmpstatus = eval {
7744         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7745         mon_cmd($vmid, "query-status");
7746     };
7747     warn "$@\n" if $@;
7748     return $qmpstatus && $qmpstatus->{status} eq "paused";
7749 }
7750
7751 sub check_volume_storage_type {
7752     my ($storecfg, $vol) = @_;
7753
7754     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7755     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7756     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7757
7758     die "storage '$storeid' does not support content-type '$vtype'\n"
7759         if !$scfg->{content}->{$vtype};
7760
7761     return 1;
7762 }
7763
7764 1;