PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RESTEnvironment qw(log_warn);
  40 use PVE::RPCEnvironment;
  41 use PVE::Storage;
  42 use PVE::SysFSTools;
  43 use PVE::Systemd;
  44 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  45
  46 use PVE::QMPClient;
  47 use PVE::QemuConfig;
  48 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  49 use PVE::QemuServer::Cloudinit;
  50 use PVE::QemuServer::CGroup;
  51 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  52 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  53 use PVE::QemuServer::Machine;
  54 use PVE::QemuServer::Memory;
  55 use PVE::QemuServer::Monitor qw(mon_cmd);
  56 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  57 use PVE::QemuServer::USB qw(parse_usb_device);
  58
  59 my $have_sdn;
  60 eval {
  61     require PVE::Network::SDN::Zones;
  62     $have_sdn = 1;
  63 };
  64
  65 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  66 my $OVMF = {
  67     x86_64 => {
  68         '4m-no-smm' => [
  69             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  70             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  71         ],
  72         '4m-no-smm-ms' => [
  73             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  74             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  75         ],
  76         '4m' => [
  77             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  78             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  79         ],
  80         '4m-ms' => [
  81             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  82             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  83         ],
  84         default => [
  85             "$EDK2_FW_BASE/OVMF_CODE.fd",
  86             "$EDK2_FW_BASE/OVMF_VARS.fd",
  87         ],
  88     },
  89     aarch64 => {
  90         default => [
  91             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  92             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  93         ],
  94     },
  95 };
  96
  97 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  98
  99 # Note about locking: we use flock on the config file protect against concurent actions.
 100 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 101 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 102 # But you can ignore this kind of lock with the --skiplock flag.
 103
 104 cfs_register_file('/qemu-server/',
 105                   \&parse_vm_config,
 106                   \&write_vm_config);
 107
 108 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 109     description => "Some command save/restore state from this location.",
 110     type => 'string',
 111     maxLength => 128,
 112     optional => 1,
 113 });
 114
 115 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 116         description => "Specifies the Qemu machine type.",
 117         type => 'string',
 118         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 119         maxLength => 40,
 120         optional => 1,
 121 });
 122
 123 #no warnings 'redefine';
 124
 125 my $nodename_cache;
 126 sub nodename {
 127     $nodename_cache //= PVE::INotify::nodename();
 128     return $nodename_cache;
 129 }
 130
 131 my $watchdog_fmt = {
 132     model => {
 133         default_key => 1,
 134         type => 'string',
 135         enum => [qw(i6300esb ib700)],
 136         description => "Watchdog type to emulate.",
 137         default => 'i6300esb',
 138         optional => 1,
 139     },
 140     action => {
 141         type => 'string',
 142         enum => [qw(reset shutdown poweroff pause debug none)],
 143         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 144         optional => 1,
 145     },
 146 };
 147 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 148
 149 my $agent_fmt = {
 150     enabled => {
 151         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 152         type => 'boolean',
 153         default => 0,
 154         default_key => 1,
 155     },
 156     fstrim_cloned_disks => {
 157         description => "Run fstrim after moving a disk or migrating the VM.",
 158         type => 'boolean',
 159         optional => 1,
 160         default => 0
 161     },
 162     type => {
 163         description => "Select the agent type",
 164         type => 'string',
 165         default => 'virtio',
 166         optional => 1,
 167         enum => [qw(virtio isa)],
 168     },
 169 };
 170
 171 my $vga_fmt = {
 172     type => {
 173         description => "Select the VGA type.",
 174         type => 'string',
 175         default => 'std',
 176         optional => 1,
 177         default_key => 1,
 178         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 179     },
 180     memory => {
 181         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 182         type => 'integer',
 183         optional => 1,
 184         minimum => 4,
 185         maximum => 512,
 186     },
 187 };
 188
 189 my $ivshmem_fmt = {
 190     size => {
 191         type => 'integer',
 192         minimum => 1,
 193         description => "The size of the file in MB.",
 194     },
 195     name => {
 196         type => 'string',
 197         pattern => '[a-zA-Z0-9\-]+',
 198         optional => 1,
 199         format_description => 'string',
 200         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 201     },
 202 };
 203
 204 my $audio_fmt = {
 205     device => {
 206         type => 'string',
 207         enum => [qw(ich9-intel-hda intel-hda AC97)],
 208         description =>  "Configure an audio device."
 209     },
 210     driver =>  {
 211         type => 'string',
 212         enum => ['spice', 'none'],
 213         default => 'spice',
 214         optional => 1,
 215         description => "Driver backend for the audio device."
 216     },
 217 };
 218
 219 my $spice_enhancements_fmt = {
 220     foldersharing => {
 221         type => 'boolean',
 222         optional => 1,
 223         default => '0',
 224         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 225     },
 226     videostreaming =>  {
 227         type => 'string',
 228         enum => ['off', 'all', 'filter'],
 229         default => 'off',
 230         optional => 1,
 231         description => "Enable video streaming. Uses compression for detected video streams."
 232     },
 233 };
 234
 235 my $rng_fmt = {
 236     source => {
 237         type => 'string',
 238         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 239         default_key => 1,
 240         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 241             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 242             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 243             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 244             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 245             ." a hardware RNG from the host.",
 246     },
 247     max_bytes => {
 248         type => 'integer',
 249         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 250             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 251             ." `0` to disable limiting (potentially dangerous!).",
 252         optional => 1,
 253
 254         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 255         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 256         # reading from /dev/urandom
 257         default => 1024,
 258     },
 259     period => {
 260         type => 'integer',
 261         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 262             ." the guest to retrieve another 'max_bytes' of entropy.",
 263         optional => 1,
 264         default => 1000,
 265     },
 266 };
 267
 268 my $meta_info_fmt = {
 269     'ctime' => {
 270         type => 'integer',
 271         description => "The guest creation timestamp as UNIX epoch time",
 272         minimum => 0,
 273         optional => 1,
 274     },
 275     'creation-qemu' => {
 276         type => 'string',
 277         description => "The QEMU (machine) version from the time this VM was created.",
 278         pattern => '\d+(\.\d+)+',
 279         optional => 1,
 280     },
 281 };
 282
 283 my $confdesc = {
 284     onboot => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Specifies whether a VM will be started during system bootup.",
 288         default => 0,
 289     },
 290     autostart => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Automatic restart after crash (currently ignored).",
 294         default => 0,
 295     },
 296     hotplug => {
 297         optional => 1,
 298         type => 'string', format => 'pve-hotplug-features',
 299         description => "Selectively enable hotplug features. This is a comma separated list of"
 300             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 301             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 302         default => 'network,disk,usb',
 303     },
 304     reboot => {
 305         optional => 1,
 306         type => 'boolean',
 307         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 308         default => 1,
 309     },
 310     lock => {
 311         optional => 1,
 312         type => 'string',
 313         description => "Lock/unlock the VM.",
 314         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 315     },
 316     cpulimit => {
 317         optional => 1,
 318         type => 'number',
 319         description => "Limit of CPU usage.",
 320         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 321             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 322         minimum => 0,
 323         maximum => 128,
 324         default => 0,
 325     },
 326     cpuunits => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 330         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 331             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 332             ." weights of all the other running VMs.",
 333         minimum => 1,
 334         maximum => 262144,
 335         default => 'cgroup v1: 1024, cgroup v2: 100',
 336     },
 337     memory => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 341             ." you use the balloon device.",
 342         minimum => 16,
 343         default => 512,
 344     },
 345     balloon => {
 346         optional => 1,
 347         type => 'integer',
 348         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 349         minimum => 0,
 350     },
 351     shares => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 355             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 356             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 357         minimum => 0,
 358         maximum => 50000,
 359         default => 1000,
 360     },
 361     keyboard => {
 362         optional => 1,
 363         type => 'string',
 364         description => "Keyboard layout for VNC server. This option is generally not required and"
 365         ." is often better handled from within the guest OS.",
 366         enum => PVE::Tools::kvmkeymaplist(),
 367         default => undef,
 368     },
 369     name => {
 370         optional => 1,
 371         type => 'string', format => 'dns-name',
 372         description => "Set a name for the VM. Only used on the configuration web interface.",
 373     },
 374     scsihw => {
 375         optional => 1,
 376         type => 'string',
 377         description => "SCSI controller model",
 378         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 379         default => 'lsi',
 380     },
 381     description => {
 382         optional => 1,
 383         type => 'string',
 384         description => "Description for the VM. Shown in the web-interface VM's summary."
 385             ." This is saved as comment inside the configuration file.",
 386         maxLength => 1024 * 8,
 387     },
 388     ostype => {
 389         optional => 1,
 390         type => 'string',
 391         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 392         description => "Specify guest operating system.",
 393         verbose_description => <<EODESC,
 394 Specify guest operating system. This is used to enable special
 395 optimization/features for specific operating systems:
 396
 397 [horizontal]
 398 other;; unspecified OS
 399 wxp;; Microsoft Windows XP
 400 w2k;; Microsoft Windows 2000
 401 w2k3;; Microsoft Windows 2003
 402 w2k8;; Microsoft Windows 2008
 403 wvista;; Microsoft Windows Vista
 404 win7;; Microsoft Windows 7
 405 win8;; Microsoft Windows 8/2012/2012r2
 406 win10;; Microsoft Windows 10/2016/2019
 407 win11;; Microsoft Windows 11/2022
 408 l24;; Linux 2.4 Kernel
 409 l26;; Linux 2.6 - 5.X Kernel
 410 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 411 EODESC
 412     },
 413     boot => {
 414         optional => 1,
 415         type => 'string', format => 'pve-qm-boot',
 416         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 417             ." key or 'legacy=' is deprecated.",
 418     },
 419     bootdisk => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-bootdisk',
 422         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 423         pattern => '(ide|sata|scsi|virtio)\d+',
 424     },
 425     smp => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of CPUs. Please use option -sockets instead.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     sockets => {
 433         optional => 1,
 434         type => 'integer',
 435         description => "The number of CPU sockets.",
 436         minimum => 1,
 437         default => 1,
 438     },
 439     cores => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "The number of cores per socket.",
 443         minimum => 1,
 444         default => 1,
 445     },
 446     numa => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable NUMA.",
 450         default => 0,
 451     },
 452     hugepages => {
 453         optional => 1,
 454         type => 'string',
 455         description => "Enable/disable hugepages memory.",
 456         enum => [qw(any 2 1024)],
 457     },
 458     keephugepages => {
 459         optional => 1,
 460         type => 'boolean',
 461         default => 0,
 462         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 463             ." after VM shutdown and can be used for subsequent starts.",
 464     },
 465     vcpus => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "Number of hotplugged vcpus.",
 469         minimum => 1,
 470         default => 0,
 471     },
 472     acpi => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable ACPI.",
 476         default => 1,
 477     },
 478     agent => {
 479         optional => 1,
 480         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 481         type => 'string',
 482         format => $agent_fmt,
 483     },
 484     kvm => {
 485         optional => 1,
 486         type => 'boolean',
 487         description => "Enable/disable KVM hardware virtualization.",
 488         default => 1,
 489     },
 490     tdf => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable time drift fix.",
 494         default => 0,
 495     },
 496     localtime => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 500             ." the `ostype` indicates a Microsoft Windows OS.",
 501     },
 502     freeze => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 506     },
 507     vga => {
 508         optional => 1,
 509         type => 'string', format => $vga_fmt,
 510         description => "Configure the VGA hardware.",
 511         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 512             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 513             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 514             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 515             ." display server. For win* OS you can select how many independent displays you want,"
 516             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 517             ." using a serial device as terminal.",
 518     },
 519     watchdog => {
 520         optional => 1,
 521         type => 'string', format => 'pve-qm-watchdog',
 522         description => "Create a virtual hardware watchdog device.",
 523         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 524             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 525             ." else the watchdog will reset the guest (or execute the respective action specified)",
 526     },
 527     startdate => {
 528         optional => 1,
 529         type => 'string',
 530         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 531         description => "Set the initial date of the real time clock. Valid format for date are:"
 532             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 533         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 534         default => 'now',
 535     },
 536     startup =>  get_standard_option('pve-startup-order'),
 537     template => {
 538         optional => 1,
 539         type => 'boolean',
 540         description => "Enable/disable Template.",
 541         default => 0,
 542     },
 543     args => {
 544         optional => 1,
 545         type => 'string',
 546         description => "Arbitrary arguments passed to kvm.",
 547         verbose_description => <<EODESCR,
 548 Arbitrary arguments passed to kvm, for example:
 549
 550 args: -no-reboot -no-hpet
 551
 552 NOTE: this option is for experts only.
 553 EODESCR
 554     },
 555     tablet => {
 556         optional => 1,
 557         type => 'boolean',
 558         default => 1,
 559         description => "Enable/disable the USB tablet device.",
 560         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 561             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 562             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 563             ." may consider disabling this to save some context switches. This is turned off by"
 564             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 565     },
 566     migrate_speed => {
 567         optional => 1,
 568         type => 'integer',
 569         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 570         minimum => 0,
 571         default => 0,
 572     },
 573     migrate_downtime => {
 574         optional => 1,
 575         type => 'number',
 576         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 577         minimum => 0,
 578         default => 0.1,
 579     },
 580     cdrom => {
 581         optional => 1,
 582         type => 'string', format => 'pve-qm-ide',
 583         typetext => '<volume>',
 584         description => "This is an alias for option -ide2",
 585     },
 586     cpu => {
 587         optional => 1,
 588         description => "Emulated CPU type.",
 589         type => 'string',
 590         format => 'pve-vm-cpu-conf',
 591     },
 592     parent => get_standard_option('pve-snapshot-name', {
 593         optional => 1,
 594         description => "Parent snapshot name. This is used internally, and should not be modified.",
 595     }),
 596     snaptime => {
 597         optional => 1,
 598         description => "Timestamp for snapshots.",
 599         type => 'integer',
 600         minimum => 0,
 601     },
 602     vmstate => {
 603         optional => 1,
 604         type => 'string', format => 'pve-volume-id',
 605         description => "Reference to a volume which stores the VM state. This is used internally"
 606             ." for snapshots.",
 607     },
 608     vmstatestorage => get_standard_option('pve-storage-id', {
 609         description => "Default storage for VM state volumes/files.",
 610         optional => 1,
 611     }),
 612     runningmachine => get_standard_option('pve-qemu-machine', {
 613         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 614             ." for snapshots.",
 615     }),
 616     runningcpu => {
 617         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 618             ." internally for snapshots.",
 619         optional => 1,
 620         type => 'string',
 621         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 622         format_description => 'QEMU -cpu parameter'
 623     },
 624     machine => get_standard_option('pve-qemu-machine'),
 625     arch => {
 626         description => "Virtual processor architecture. Defaults to the host.",
 627         optional => 1,
 628         type => 'string',
 629         enum => [qw(x86_64 aarch64)],
 630     },
 631     smbios1 => {
 632         description => "Specify SMBIOS type 1 fields.",
 633         type => 'string', format => 'pve-qm-smbios1',
 634         maxLength => 512,
 635         optional => 1,
 636     },
 637     protection => {
 638         optional => 1,
 639         type => 'boolean',
 640         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 641             ." remove disk operations.",
 642         default => 0,
 643     },
 644     bios => {
 645         optional => 1,
 646         type => 'string',
 647         enum => [ qw(seabios ovmf) ],
 648         description => "Select BIOS implementation.",
 649         default => 'seabios',
 650     },
 651     vmgenid => {
 652         type => 'string',
 653         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 654         format_description => 'UUID',
 655         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 656             ." to disable explicitly.",
 657         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 658             ." value identifier to the guest OS. This allows to notify the guest operating system"
 659             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 660             ." execution or creation from a template). The guest operating system notices the"
 661             ." change, and is then able to react as appropriate by marking its copies of"
 662             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 663             ."Note that auto-creation only works when done through API/CLI create or update methods"
 664             .", but not when manually editing the config file.",
 665         default => "1 (autogenerated)",
 666         optional => 1,
 667     },
 668     hookscript => {
 669         type => 'string',
 670         format => 'pve-volume-id',
 671         optional => 1,
 672         description => "Script that will be executed during various steps in the vms lifetime.",
 673     },
 674     ivshmem => {
 675         type => 'string',
 676         format => $ivshmem_fmt,
 677         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 678             ." the host.",
 679         optional => 1,
 680     },
 681     audio0 => {
 682         type => 'string',
 683         format => $audio_fmt,
 684         description => "Configure a audio device, useful in combination with QXL/Spice.",
 685         optional => 1
 686     },
 687     spice_enhancements => {
 688         type => 'string',
 689         format => $spice_enhancements_fmt,
 690         description => "Configure additional enhancements for SPICE.",
 691         optional => 1
 692     },
 693     tags => {
 694         type => 'string', format => 'pve-tag-list',
 695         description => 'Tags of the VM. This is only meta information.',
 696         optional => 1,
 697     },
 698     rng0 => {
 699         type => 'string',
 700         format => $rng_fmt,
 701         description => "Configure a VirtIO-based Random Number Generator.",
 702         optional => 1,
 703     },
 704     meta => {
 705         type => 'string',
 706         format => $meta_info_fmt,
 707         description => "Some (read-only) meta-information about this guest.",
 708         optional => 1,
 709     },
 710 };
 711
 712 my $cicustom_fmt = {
 713     meta => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all meta data passed to the VM via"
 717             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     network => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all network data passed to the VM via'
 725             .' cloud-init.',
 726         format => 'pve-volume-id',
 727         format_description => 'volume',
 728     },
 729     user => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all user data passed to the VM via'
 733             .' cloud-init.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737     vendor => {
 738     type => 'string',
 739     optional => 1,
 740     description => 'Specify a custom file containing all vendor data passed to the VM via'
 741      .' cloud-init.',
 742     format => 'pve-volume-id',
 743     format_description => 'volume',
 744     },
 745 };
 746 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 747
 748 my $confdesc_cloudinit = {
 749     citype => {
 750         optional => 1,
 751         type => 'string',
 752         description => 'Specifies the cloud-init configuration format. The default depends on the'
 753             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 754             .' and `configdrive2` for windows.',
 755         enum => ['configdrive2', 'nocloud', 'opennebula'],
 756     },
 757     ciuser => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: User name to change ssh keys and password for instead of the"
 761             ." image's configured default user.",
 762     },
 763     cipassword => {
 764         optional => 1,
 765         type => 'string',
 766         description => 'cloud-init: Password to assign the user. Using this is generally not'
 767             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 768             .' support hashed passwords.',
 769     },
 770     cicustom => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'cloud-init: Specify custom files to replace the automatically generated'
 774             .' ones at start.',
 775         format => 'pve-qm-cicustom',
 776     },
 777     searchdomain => {
 778         optional => 1,
 779         type => 'string',
 780         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 781             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 782             .' are set.',
 783     },
 784     nameserver => {
 785         optional => 1,
 786         type => 'string', format => 'address-list',
 787         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 788             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 789             .' are set.',
 790     },
 791     sshkeys => {
 792         optional => 1,
 793         type => 'string',
 794         format => 'urlencoded',
 795         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 796     },
 797 };
 798
 799 # what about other qemu settings ?
 800 #cpu => 'string',
 801 #machine => 'string',
 802 #fda => 'file',
 803 #fdb => 'file',
 804 #mtdblock => 'file',
 805 #sd => 'file',
 806 #pflash => 'file',
 807 #snapshot => 'bool',
 808 #bootp => 'file',
 809 ##tftp => 'dir',
 810 ##smb => 'dir',
 811 #kernel => 'file',
 812 #append => 'string',
 813 #initrd => 'file',
 814 ##soundhw => 'string',
 815
 816 while (my ($k, $v) = each %$confdesc) {
 817     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 818 }
 819
 820 my $MAX_USB_DEVICES = 5;
 821 my $MAX_NETS = 32;
 822 my $MAX_SERIAL_PORTS = 4;
 823 my $MAX_PARALLEL_PORTS = 3;
 824 my $MAX_NUMA = 8;
 825
 826 my $numa_fmt = {
 827     cpus => {
 828         type => "string",
 829         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 830         description => "CPUs accessing this NUMA node.",
 831         format_description => "id[-id];...",
 832     },
 833     memory => {
 834         type => "number",
 835         description => "Amount of memory this NUMA node provides.",
 836         optional => 1,
 837     },
 838     hostnodes => {
 839         type => "string",
 840         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 841         description => "Host NUMA nodes to use.",
 842         format_description => "id[-id];...",
 843         optional => 1,
 844     },
 845     policy => {
 846         type => 'string',
 847         enum => [qw(preferred bind interleave)],
 848         description => "NUMA allocation policy.",
 849         optional => 1,
 850     },
 851 };
 852 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 853 my $numadesc = {
 854     optional => 1,
 855     type => 'string', format => $numa_fmt,
 856     description => "NUMA topology.",
 857 };
 858 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 859
 860 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 861     $confdesc->{"numa$i"} = $numadesc;
 862 }
 863
 864 my $nic_model_list = [
 865     'e1000',
 866     'e1000-82540em',
 867     'e1000-82544gc',
 868     'e1000-82545em',
 869     'e1000e',
 870     'i82551',
 871     'i82557b',
 872     'i82559er',
 873     'ne2k_isa',
 874     'ne2k_pci',
 875     'pcnet',
 876     'rtl8139',
 877     'virtio',
 878     'vmxnet3',
 879 ];
 880 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 881
 882 my $net_fmt_bridge_descr = <<__EOD__;
 883 Bridge to attach the network device to. The Proxmox VE standard bridge
 884 is called 'vmbr0'.
 885
 886 If you do not specify a bridge, we create a kvm user (NATed) network
 887 device, which provides DHCP and DNS services. The following addresses
 888 are used:
 889
 890  10.0.2.2   Gateway
 891  10.0.2.3   DNS Server
 892  10.0.2.4   SMB Server
 893
 894 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 895 __EOD__
 896
 897 my $net_fmt = {
 898     macaddr  => get_standard_option('mac-addr', {
 899         description => "MAC address. That address must be unique withing your network. This is"
 900             ." automatically generated if not specified.",
 901     }),
 902     model => {
 903         type => 'string',
 904         description => "Network Card Model. The 'virtio' model provides the best performance with"
 905             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 906             ." best to use 'e1000'.",
 907         enum => $nic_model_list,
 908         default_key => 1,
 909     },
 910     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 911     bridge => get_standard_option('pve-bridge-id', {
 912         description => $net_fmt_bridge_descr,
 913         optional => 1,
 914     }),
 915     queues => {
 916         type => 'integer',
 917         minimum => 0, maximum => 16,
 918         description => 'Number of packet queues to be used on the device.',
 919         optional => 1,
 920     },
 921     rate => {
 922         type => 'number',
 923         minimum => 0,
 924         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 925         optional => 1,
 926     },
 927     tag => {
 928         type => 'integer',
 929         minimum => 1, maximum => 4094,
 930         description => 'VLAN tag to apply to packets on this interface.',
 931         optional => 1,
 932     },
 933     trunks => {
 934         type => 'string',
 935         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 936         description => 'VLAN trunks to pass through this interface.',
 937         format_description => 'vlanid[;vlanid...]',
 938         optional => 1,
 939     },
 940     firewall => {
 941         type => 'boolean',
 942         description => 'Whether this interface should be protected by the firewall.',
 943         optional => 1,
 944     },
 945     link_down => {
 946         type => 'boolean',
 947         description => 'Whether this interface should be disconnected (like pulling the plug).',
 948         optional => 1,
 949     },
 950     mtu => {
 951         type => 'integer',
 952         minimum => 1, maximum => 65520,
 953         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 954         optional => 1,
 955     },
 956 };
 957
 958 my $netdesc = {
 959     optional => 1,
 960     type => 'string', format => $net_fmt,
 961     description => "Specify network devices.",
 962 };
 963
 964 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 965
 966 my $ipconfig_fmt = {
 967     ip => {
 968         type => 'string',
 969         format => 'pve-ipv4-config',
 970         format_description => 'IPv4Format/CIDR',
 971         description => 'IPv4 address in CIDR format.',
 972         optional => 1,
 973         default => 'dhcp',
 974     },
 975     gw => {
 976         type => 'string',
 977         format => 'ipv4',
 978         format_description => 'GatewayIPv4',
 979         description => 'Default gateway for IPv4 traffic.',
 980         optional => 1,
 981         requires => 'ip',
 982     },
 983     ip6 => {
 984         type => 'string',
 985         format => 'pve-ipv6-config',
 986         format_description => 'IPv6Format/CIDR',
 987         description => 'IPv6 address in CIDR format.',
 988         optional => 1,
 989         default => 'dhcp',
 990     },
 991     gw6 => {
 992         type => 'string',
 993         format => 'ipv6',
 994         format_description => 'GatewayIPv6',
 995         description => 'Default gateway for IPv6 traffic.',
 996         optional => 1,
 997         requires => 'ip6',
 998     },
 999 };
1000 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1001 my $ipconfigdesc = {
1002     optional => 1,
1003     type => 'string', format => 'pve-qm-ipconfig',
1004     description => <<'EODESCR',
1005 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1006
1007 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1008
1009 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1010 gateway should be provided.
1011 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1012 cloud-init 19.4 or newer.
1013
1014 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1015 dhcp on IPv4.
1016 EODESCR
1017 };
1018 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1019
1020 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1021     $confdesc->{"net$i"} = $netdesc;
1022     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1023 }
1024
1025 foreach my $key (keys %$confdesc_cloudinit) {
1026     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1027 }
1028
1029 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1030 sub verify_volume_id_or_qm_path {
1031     my ($volid, $noerr) = @_;
1032
1033     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1034
1035     return verify_volume_id_or_absolute_path($volid, $noerr);
1036 }
1037
1038 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1039 sub verify_volume_id_or_absolute_path {
1040     my ($volid, $noerr) = @_;
1041
1042     return $volid if $volid =~ m|^/|;
1043
1044     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1045     if ($@) {
1046         return if $noerr;
1047         die $@;
1048     }
1049     return $volid;
1050 }
1051
1052 my $usb_fmt = {
1053     host => {
1054         default_key => 1,
1055         type => 'string', format => 'pve-qm-usb-device',
1056         format_description => 'HOSTUSBDEVICE|spice',
1057         description => <<EODESCR,
1058 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1059
1060  'bus-port(.port)*' (decimal numbers) or
1061  'vendor_id:product_id' (hexadeciaml numbers) or
1062  'spice'
1063
1064 You can use the 'lsusb -t' command to list existing usb devices.
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 The value 'spice' can be used to add a usb redirection devices for spice.
1070 EODESCR
1071     },
1072     usb3 => {
1073         optional => 1,
1074         type => 'boolean',
1075         description => "Specifies whether if given host option is a USB3 device or port.",
1076         default => 0,
1077     },
1078 };
1079
1080 my $usbdesc = {
1081     optional => 1,
1082     type => 'string', format => $usb_fmt,
1083     description => "Configure an USB device (n is 0 to 4).",
1084 };
1085 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1086
1087 my $serialdesc = {
1088         optional => 1,
1089         type => 'string',
1090         pattern => '(/dev/.+|socket)',
1091         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1092         verbose_description =>  <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1096
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1098 use with special care.
1099
1100 CAUTION: Experimental! User reported problems with this option.
1101 EODESCR
1102 };
1103
1104 my $paralleldesc= {
1105         optional => 1,
1106         type => 'string',
1107         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1108         description =>  "Map host parallel devices (n is 0 to 2).",
1109         verbose_description =>  <<EODESCR,
1110 Map host parallel devices (n is 0 to 2).
1111
1112 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1113 machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1129 }
1130
1131 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1132     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1133 }
1134
1135 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1136     $confdesc->{"usb$i"} = $usbdesc;
1137 }
1138
1139 my $boot_fmt = {
1140     legacy => {
1141         optional => 1,
1142         default_key => 1,
1143         type => 'string',
1144         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1145                      . " Deprecated, use 'order=' instead.",
1146         pattern => '[acdn]{1,4}',
1147         format_description => "[acdn]{1,4}",
1148
1149         # note: this is also the fallback if boot: is not given at all
1150         default => 'cdn',
1151     },
1152     order => {
1153         optional => 1,
1154         type => 'string',
1155         format => 'pve-qm-bootdev-list',
1156         format_description => "device[;device...]",
1157         description => <<EODESC,
1158 The guest will attempt to boot from devices in the order they appear here.
1159
1160 Disks, optical drives and passed-through storage USB devices will be directly
1161 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1162 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1163
1164 Note that only devices in this list will be marked as bootable and thus loaded
1165 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1166 (e.g. software-raid), you need to specify all of them here.
1167
1168 Overrides the deprecated 'legacy=[acdn]*' value when given.
1169 EODESC
1170     },
1171 };
1172 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1173
1174 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1175 sub verify_bootdev {
1176     my ($dev, $noerr) = @_;
1177
1178     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1179     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1180
1181     my $check = sub {
1182         my ($base) = @_;
1183         return 0 if $dev !~ m/^$base\d+$/;
1184         return 0 if !$confdesc->{$dev};
1185         return 1;
1186     };
1187
1188     return $dev if $check->("net");
1189     return $dev if $check->("usb");
1190     return $dev if $check->("hostpci");
1191
1192     return if $noerr;
1193     die "invalid boot device '$dev'\n";
1194 }
1195
1196 sub print_bootorder {
1197     my ($devs) = @_;
1198     return "" if !@$devs;
1199     my $data = { order => join(';', @$devs) };
1200     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1201 }
1202
1203 my $kvm_api_version = 0;
1204
1205 sub kvm_version {
1206     return $kvm_api_version if $kvm_api_version;
1207
1208     open my $fh, '<', '/dev/kvm' or return;
1209
1210     # 0xae00 => KVM_GET_API_VERSION
1211     $kvm_api_version = ioctl($fh, 0xae00, 0);
1212     close($fh);
1213
1214     return $kvm_api_version;
1215 }
1216
1217 my $kvm_user_version = {};
1218 my $kvm_mtime = {};
1219
1220 sub kvm_user_version {
1221     my ($binary) = @_;
1222
1223     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1224     my $st = stat($binary);
1225
1226     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1227     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1228         $cachedmtime == $st->mtime;
1229
1230     $kvm_user_version->{$binary} = 'unknown';
1231     $kvm_mtime->{$binary} = $st->mtime;
1232
1233     my $code = sub {
1234         my $line = shift;
1235         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1236             $kvm_user_version->{$binary} = $2;
1237         }
1238     };
1239
1240     eval { run_command([$binary, '--version'], outfunc => $code); };
1241     warn $@ if $@;
1242
1243     return $kvm_user_version->{$binary};
1244
1245 }
1246 my sub extract_version {
1247     my ($machine_type, $version) = @_;
1248     $version = kvm_user_version() if !defined($version);
1249     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1250 }
1251
1252 sub kernel_has_vhost_net {
1253     return -c '/dev/vhost-net';
1254 }
1255
1256 sub option_exists {
1257     my $key = shift;
1258     return defined($confdesc->{$key});
1259 }
1260
1261 my $cdrom_path;
1262 sub get_cdrom_path {
1263
1264     return  $cdrom_path if $cdrom_path;
1265
1266     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1267     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1268     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1269 }
1270
1271 sub get_iso_path {
1272     my ($storecfg, $vmid, $cdrom) = @_;
1273
1274     if ($cdrom eq 'cdrom') {
1275         return get_cdrom_path();
1276     } elsif ($cdrom eq 'none') {
1277         return '';
1278     } elsif ($cdrom =~ m|^/|) {
1279         return $cdrom;
1280     } else {
1281         return PVE::Storage::path($storecfg, $cdrom);
1282     }
1283 }
1284
1285 # try to convert old style file names to volume IDs
1286 sub filename_to_volume_id {
1287     my ($vmid, $file, $media) = @_;
1288
1289      if (!($file eq 'none' || $file eq 'cdrom' ||
1290           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1291
1292         return if $file =~ m|/|;
1293
1294         if ($media && $media eq 'cdrom') {
1295             $file = "local:iso/$file";
1296         } else {
1297             $file = "local:$vmid/$file";
1298         }
1299     }
1300
1301     return $file;
1302 }
1303
1304 sub verify_media_type {
1305     my ($opt, $vtype, $media) = @_;
1306
1307     return if !$media;
1308
1309     my $etype;
1310     if ($media eq 'disk') {
1311         $etype = 'images';
1312     } elsif ($media eq 'cdrom') {
1313         $etype = 'iso';
1314     } else {
1315         die "internal error";
1316     }
1317
1318     return if ($vtype eq $etype);
1319
1320     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1321 }
1322
1323 sub cleanup_drive_path {
1324     my ($opt, $storecfg, $drive) = @_;
1325
1326     # try to convert filesystem paths to volume IDs
1327
1328     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1329         ($drive->{file} !~ m|^/dev/.+|) &&
1330         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1331         ($drive->{file} !~ m/^\d+$/)) {
1332         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1333         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1334             if !$vtype;
1335         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1336         verify_media_type($opt, $vtype, $drive->{media});
1337         $drive->{file} = $volid;
1338     }
1339
1340     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1341 }
1342
1343 sub parse_hotplug_features {
1344     my ($data) = @_;
1345
1346     my $res = {};
1347
1348     return $res if $data eq '0';
1349
1350     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1351
1352     foreach my $feature (PVE::Tools::split_list($data)) {
1353         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1354             $res->{$1} = 1;
1355         } else {
1356             die "invalid hotplug feature '$feature'\n";
1357         }
1358     }
1359     return $res;
1360 }
1361
1362 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1363 sub pve_verify_hotplug_features {
1364     my ($value, $noerr) = @_;
1365
1366     return $value if parse_hotplug_features($value);
1367
1368     return if $noerr;
1369
1370     die "unable to parse hotplug option\n";
1371 }
1372
1373 sub scsi_inquiry {
1374     my($fh, $noerr) = @_;
1375
1376     my $SG_IO = 0x2285;
1377     my $SG_GET_VERSION_NUM = 0x2282;
1378
1379     my $versionbuf = "\x00" x 8;
1380     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1381     if (!$ret) {
1382         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1383         return;
1384     }
1385     my $version = unpack("I", $versionbuf);
1386     if ($version < 30000) {
1387         die "scsi generic interface too old\n"  if !$noerr;
1388         return;
1389     }
1390
1391     my $buf = "\x00" x 36;
1392     my $sensebuf = "\x00" x 8;
1393     my $cmd = pack("C x3 C x1", 0x12, 36);
1394
1395     # see /usr/include/scsi/sg.h
1396     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1397
1398     my $packet = pack(
1399         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1400     );
1401
1402     $ret = ioctl($fh, $SG_IO, $packet);
1403     if (!$ret) {
1404         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1405         return;
1406     }
1407
1408     my @res = unpack($sg_io_hdr_t, $packet);
1409     if ($res[17] || $res[18]) {
1410         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my $res = {};
1415     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1416
1417     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1418     $res->{type} &= 0x1F;
1419
1420     return $res;
1421 }
1422
1423 sub path_is_scsi {
1424     my ($path) = @_;
1425
1426     my $fh = IO::File->new("+<$path") || return;
1427     my $res = scsi_inquiry($fh, 1);
1428     close($fh);
1429
1430     return $res;
1431 }
1432
1433 sub print_tabletdevice_full {
1434     my ($conf, $arch) = @_;
1435
1436     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1437
1438     # we use uhci for old VMs because tablet driver was buggy in older qemu
1439     my $usbbus;
1440     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1441         $usbbus = 'ehci';
1442     } else {
1443         $usbbus = 'uhci';
1444     }
1445
1446     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1447 }
1448
1449 sub print_keyboarddevice_full {
1450     my ($conf, $arch) = @_;
1451
1452     return if $arch ne 'aarch64';
1453
1454     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1455 }
1456
1457 my sub get_drive_id {
1458     my ($drive) = @_;
1459     return "$drive->{interface}$drive->{index}";
1460 }
1461
1462 sub print_drivedevice_full {
1463     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1464
1465     my $device = '';
1466     my $maxdev = 0;
1467
1468     my $drive_id = get_drive_id($drive);
1469     if ($drive->{interface} eq 'virtio') {
1470         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1471         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1472         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1473     } elsif ($drive->{interface} eq 'scsi') {
1474
1475         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1476         my $unit = $drive->{index} % $maxdev;
1477         my $devicetype = 'hd';
1478         my $path = '';
1479         if (drive_is_cdrom($drive)) {
1480             $devicetype = 'cd';
1481         } else {
1482             if ($drive->{file} =~ m|^/|) {
1483                 $path = $drive->{file};
1484                 if (my $info = path_is_scsi($path)) {
1485                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1486                         $devicetype = 'block';
1487                     } elsif ($info->{type} == 1) { # tape
1488                         $devicetype = 'generic';
1489                     }
1490                 }
1491             } else {
1492                  $path = PVE::Storage::path($storecfg, $drive->{file});
1493             }
1494
1495             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1496             my $version = extract_version($machine_type, kvm_user_version());
1497             if ($path =~ m/^iscsi\:\/\// &&
1498                !min_version($version, 4, 1)) {
1499                 $devicetype = 'generic';
1500             }
1501         }
1502
1503         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1504             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1505         } else {
1506             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1507                 .",lun=$drive->{index}";
1508         }
1509         $device .= ",drive=drive-$drive_id,id=$drive_id";
1510
1511         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1512             $device .= ",rotation_rate=1";
1513         }
1514         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1515
1516     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1517         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1518         my $controller = int($drive->{index} / $maxdev);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1521
1522         $device = "ide-$devicetype";
1523         if ($drive->{interface} eq 'ide') {
1524             $device .= ",bus=ide.$controller,unit=$unit";
1525         } else {
1526             $device .= ",bus=ahci$controller.$unit";
1527         }
1528         $device .= ",drive=drive-$drive_id,id=$drive_id";
1529
1530         if ($devicetype eq 'hd') {
1531             if (my $model = $drive->{model}) {
1532                 $model = URI::Escape::uri_unescape($model);
1533                 $device .= ",model=$model";
1534             }
1535             if ($drive->{ssd}) {
1536                 $device .= ",rotation_rate=1";
1537             }
1538         }
1539         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1540     } elsif ($drive->{interface} eq 'usb') {
1541         die "implement me";
1542         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1543     } else {
1544         die "unsupported interface type";
1545     }
1546
1547     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1548
1549     if (my $serial = $drive->{serial}) {
1550         $serial = URI::Escape::uri_unescape($serial);
1551         $device .= ",serial=$serial";
1552     }
1553
1554
1555     return $device;
1556 }
1557
1558 sub get_initiator_name {
1559     my $initiator;
1560
1561     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1562     while (defined(my $line = <$fh>)) {
1563         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1564         $initiator = $1;
1565         last;
1566     }
1567     $fh->close();
1568
1569     return $initiator;
1570 }
1571
1572 sub print_drive_commandline_full {
1573     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1574
1575     my $path;
1576     my $volid = $drive->{file};
1577     my $format = $drive->{format};
1578     my $drive_id = get_drive_id($drive);
1579
1580     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1581     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1582
1583     if (drive_is_cdrom($drive)) {
1584         $path = get_iso_path($storecfg, $vmid, $volid);
1585         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1586     } else {
1587         if ($storeid) {
1588             $path = PVE::Storage::path($storecfg, $volid);
1589             $format //= qemu_img_format($scfg, $volname);
1590         } else {
1591             $path = $volid;
1592             $format //= "raw";
1593         }
1594    }
1595
1596    my $is_rbd = $path =~ m/^rbd:/;
1597
1598     my $opts = '';
1599     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1600     foreach my $o (@qemu_drive_options) {
1601         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1602     }
1603
1604     # snapshot only accepts on|off
1605     if (defined($drive->{snapshot})) {
1606         my $v = $drive->{snapshot} ? 'on' : 'off';
1607         $opts .= ",snapshot=$v";
1608     }
1609
1610     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1611         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1612     }
1613
1614     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1615         my ($dir, $qmpname) = @$type;
1616         if (my $v = $drive->{"mbps$dir"}) {
1617             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1618         }
1619         if (my $v = $drive->{"mbps${dir}_max"}) {
1620             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1621         }
1622         if (my $v = $drive->{"bps${dir}_max_length"}) {
1623             $opts .= ",throttling.bps$qmpname-max-length=$v";
1624         }
1625         if (my $v = $drive->{"iops${dir}"}) {
1626             $opts .= ",throttling.iops$qmpname=$v";
1627         }
1628         if (my $v = $drive->{"iops${dir}_max"}) {
1629             $opts .= ",throttling.iops$qmpname-max=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}_max_length"}) {
1632             $opts .= ",throttling.iops$qmpname-max-length=$v";
1633         }
1634     }
1635
1636     if ($pbs_name) {
1637         $format = "rbd" if $is_rbd;
1638         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1639             if !$format;
1640         $opts .= ",format=alloc-track,file.driver=$format";
1641     } elsif ($format) {
1642         $opts .= ",format=$format";
1643     }
1644
1645     my $cache_direct = 0;
1646
1647     if (my $cache = $drive->{cache}) {
1648         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1649     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1650         $opts .= ",cache=none";
1651         $cache_direct = 1;
1652     }
1653
1654     # io_uring with cache mode writeback or writethrough on krbd will hang...
1655     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1656
1657     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1658     # sometimes, just plain disable...
1659     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1660
1661     # io_uring causes problems when used with CIFS since kernel 5.15
1662     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1663     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1664
1665     if (!$drive->{aio}) {
1666         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1667             # io_uring supports all cache modes
1668             $opts .= ",aio=io_uring";
1669         } else {
1670             # aio native works only with O_DIRECT
1671             if($cache_direct) {
1672                 $opts .= ",aio=native";
1673             } else {
1674                 $opts .= ",aio=threads";
1675             }
1676         }
1677     }
1678
1679     if (!drive_is_cdrom($drive)) {
1680         my $detectzeroes;
1681         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1682             $detectzeroes = 'off';
1683         } elsif ($drive->{discard}) {
1684             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1685         } else {
1686             # This used to be our default with discard not being specified:
1687             $detectzeroes = 'on';
1688         }
1689
1690         # note: 'detect-zeroes' works per blockdev and we want it to persist
1691         # after the alloc-track is removed, so put it on 'file' directly
1692         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1693         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1694     }
1695
1696     if ($pbs_name) {
1697         $opts .= ",backing=$pbs_name";
1698         $opts .= ",auto-remove=on";
1699     }
1700
1701     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1702     my $file_param = "file";
1703     if ($pbs_name) {
1704         # non-rbd drivers require the underlying file to be a seperate block
1705         # node, so add a second .file indirection
1706         $file_param .= ".file" if !$is_rbd;
1707         $file_param .= ".filename";
1708     }
1709     my $pathinfo = $path ? "$file_param=$path," : '';
1710
1711     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1712 }
1713
1714 sub print_pbs_blockdev {
1715     my ($pbs_conf, $pbs_name) = @_;
1716     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1717     $blockdev .= ",repository=$pbs_conf->{repository}";
1718     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1719     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1720     $blockdev .= ",archive=$pbs_conf->{archive}";
1721     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1722     return $blockdev;
1723 }
1724
1725 sub print_netdevice_full {
1726     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1727
1728     my $device = $net->{model};
1729     if ($net->{model} eq 'virtio') {
1730          $device = 'virtio-net-pci';
1731      };
1732
1733     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1734     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1735     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1736         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1737         # and out of each queue plus one config interrupt and control vector queue
1738         my $vectors = $net->{queues} * 2 + 2;
1739         $tmpstr .= ",vectors=$vectors,mq=on";
1740     }
1741     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1742
1743     if (my $mtu = $net->{mtu}) {
1744         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1745             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1746             if ($mtu == 1) {
1747                  $mtu = $bridge_mtu;
1748             } elsif ($mtu < 576) {
1749                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1750             } elsif ($mtu > $bridge_mtu) {
1751                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1752             }
1753             $tmpstr .= ",host_mtu=$mtu";
1754         } else {
1755             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1756         }
1757     }
1758
1759     if ($use_old_bios_files) {
1760         my $romfile;
1761         if ($device eq 'virtio-net-pci') {
1762             $romfile = 'pxe-virtio.rom';
1763         } elsif ($device eq 'e1000') {
1764             $romfile = 'pxe-e1000.rom';
1765         } elsif ($device eq 'e1000e') {
1766             $romfile = 'pxe-e1000e.rom';
1767         } elsif ($device eq 'ne2k') {
1768             $romfile = 'pxe-ne2k_pci.rom';
1769         } elsif ($device eq 'pcnet') {
1770             $romfile = 'pxe-pcnet.rom';
1771         } elsif ($device eq 'rtl8139') {
1772             $romfile = 'pxe-rtl8139.rom';
1773         }
1774         $tmpstr .= ",romfile=$romfile" if $romfile;
1775     }
1776
1777     return $tmpstr;
1778 }
1779
1780 sub print_netdev_full {
1781     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1782
1783     my $i = '';
1784     if ($netid =~ m/^net(\d+)$/) {
1785         $i = int($1);
1786     }
1787
1788     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1789
1790     my $ifname = "tap${vmid}i$i";
1791
1792     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1793     die "interface name '$ifname' is too long (max 15 character)\n"
1794         if length($ifname) >= 16;
1795
1796     my $vhostparam = '';
1797     if (is_native($arch)) {
1798         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1799     }
1800
1801     my $vmname = $conf->{name} || "vm$vmid";
1802
1803     my $netdev = "";
1804     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1805
1806     if ($net->{bridge}) {
1807         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1808             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1809     } else {
1810         $netdev = "type=user,id=$netid,hostname=$vmname";
1811     }
1812
1813     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1814
1815     return $netdev;
1816 }
1817
1818 my $vga_map = {
1819     'cirrus' => 'cirrus-vga',
1820     'std' => 'VGA',
1821     'vmware' => 'vmware-svga',
1822     'virtio' => 'virtio-vga',
1823     'virtio-gl' => 'virtio-vga-gl',
1824 };
1825
1826 sub print_vga_device {
1827     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1828
1829     my $type = $vga_map->{$vga->{type}};
1830     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1831         $type = 'virtio-gpu';
1832     }
1833     my $vgamem_mb = $vga->{memory};
1834
1835     my $max_outputs = '';
1836     if ($qxlnum) {
1837         $type = $id ? 'qxl' : 'qxl-vga';
1838
1839         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1840             # set max outputs so linux can have up to 4 qxl displays with one device
1841             if (min_version($machine_version, 4, 1)) {
1842                 $max_outputs = ",max_outputs=4";
1843             }
1844         }
1845     }
1846
1847     die "no devicetype for $vga->{type}\n" if !$type;
1848
1849     my $memory = "";
1850     if ($vgamem_mb) {
1851         if ($vga->{type} =~ /^virtio/) {
1852             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1853             $memory = ",max_hostmem=$bytes";
1854         } elsif ($qxlnum) {
1855             # from https://www.spice-space.org/multiple-monitors.html
1856             $memory = ",vgamem_mb=$vga->{memory}";
1857             my $ram = $vgamem_mb * 4;
1858             my $vram = $vgamem_mb * 2;
1859             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1860         } else {
1861             $memory = ",vgamem_mb=$vga->{memory}";
1862         }
1863     } elsif ($qxlnum && $id) {
1864         $memory = ",ram_size=67108864,vram_size=33554432";
1865     }
1866
1867     my $edidoff = "";
1868     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1869         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1870     }
1871
1872     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1873     my $vgaid = "vga" . ($id // '');
1874     my $pciaddr;
1875     if ($q35 && $vgaid eq 'vga') {
1876         # the first display uses pcie.0 bus on q35 machines
1877         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1878     } else {
1879         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1880     }
1881
1882     if ($vga->{type} eq 'virtio-gl') {
1883         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1884         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1885             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1886
1887         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1888             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1889     }
1890
1891     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1892 }
1893
1894 sub parse_number_sets {
1895     my ($set) = @_;
1896     my $res = [];
1897     foreach my $part (split(/;/, $set)) {
1898         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1899             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1900             push @$res, [ $1, $2 ];
1901         } else {
1902             die "invalid range: $part\n";
1903         }
1904     }
1905     return $res;
1906 }
1907
1908 sub parse_numa {
1909     my ($data) = @_;
1910
1911     my $res = parse_property_string($numa_fmt, $data);
1912     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1913     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1914     return $res;
1915 }
1916
1917 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1918 sub parse_net {
1919     my ($data) = @_;
1920
1921     my $res = eval { parse_property_string($net_fmt, $data) };
1922     if ($@) {
1923         warn $@;
1924         return;
1925     }
1926     if (!defined($res->{macaddr})) {
1927         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1928         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1929     }
1930     return $res;
1931 }
1932
1933 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1934 sub parse_ipconfig {
1935     my ($data) = @_;
1936
1937     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1938     if ($@) {
1939         warn $@;
1940         return;
1941     }
1942
1943     if ($res->{gw} && !$res->{ip}) {
1944         warn 'gateway specified without specifying an IP address';
1945         return;
1946     }
1947     if ($res->{gw6} && !$res->{ip6}) {
1948         warn 'IPv6 gateway specified without specifying an IPv6 address';
1949         return;
1950     }
1951     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1952         warn 'gateway specified together with DHCP';
1953         return;
1954     }
1955     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1956         # gw6 + auto/dhcp
1957         warn "IPv6 gateway specified together with $res->{ip6} address";
1958         return;
1959     }
1960
1961     if (!$res->{ip} && !$res->{ip6}) {
1962         return { ip => 'dhcp', ip6 => 'dhcp' };
1963     }
1964
1965     return $res;
1966 }
1967
1968 sub print_net {
1969     my $net = shift;
1970
1971     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1972 }
1973
1974 sub add_random_macs {
1975     my ($settings) = @_;
1976
1977     foreach my $opt (keys %$settings) {
1978         next if $opt !~ m/^net(\d+)$/;
1979         my $net = parse_net($settings->{$opt});
1980         next if !$net;
1981         $settings->{$opt} = print_net($net);
1982     }
1983 }
1984
1985 sub vm_is_volid_owner {
1986     my ($storecfg, $vmid, $volid) = @_;
1987
1988     if ($volid !~  m|^/|) {
1989         my ($path, $owner);
1990         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1991         if ($owner && ($owner == $vmid)) {
1992             return 1;
1993         }
1994     }
1995
1996     return;
1997 }
1998
1999 sub vmconfig_register_unused_drive {
2000     my ($storecfg, $vmid, $conf, $drive) = @_;
2001
2002     if (drive_is_cloudinit($drive)) {
2003         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2004         warn $@ if $@;
2005         delete $conf->{cloudinit};
2006     } elsif (!drive_is_cdrom($drive)) {
2007         my $volid = $drive->{file};
2008         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2009             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2010         }
2011     }
2012 }
2013
2014 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2015 my $smbios1_fmt = {
2016     uuid => {
2017         type => 'string',
2018         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2019         format_description => 'UUID',
2020         description => "Set SMBIOS1 UUID.",
2021         optional => 1,
2022     },
2023     version => {
2024         type => 'string',
2025         pattern => '[A-Za-z0-9+\/]+={0,2}',
2026         format_description => 'Base64 encoded string',
2027         description => "Set SMBIOS1 version.",
2028         optional => 1,
2029     },
2030     serial => {
2031         type => 'string',
2032         pattern => '[A-Za-z0-9+\/]+={0,2}',
2033         format_description => 'Base64 encoded string',
2034         description => "Set SMBIOS1 serial number.",
2035         optional => 1,
2036     },
2037     manufacturer => {
2038         type => 'string',
2039         pattern => '[A-Za-z0-9+\/]+={0,2}',
2040         format_description => 'Base64 encoded string',
2041         description => "Set SMBIOS1 manufacturer.",
2042         optional => 1,
2043     },
2044     product => {
2045         type => 'string',
2046         pattern => '[A-Za-z0-9+\/]+={0,2}',
2047         format_description => 'Base64 encoded string',
2048         description => "Set SMBIOS1 product ID.",
2049         optional => 1,
2050     },
2051     sku => {
2052         type => 'string',
2053         pattern => '[A-Za-z0-9+\/]+={0,2}',
2054         format_description => 'Base64 encoded string',
2055         description => "Set SMBIOS1 SKU string.",
2056         optional => 1,
2057     },
2058     family => {
2059         type => 'string',
2060         pattern => '[A-Za-z0-9+\/]+={0,2}',
2061         format_description => 'Base64 encoded string',
2062         description => "Set SMBIOS1 family string.",
2063         optional => 1,
2064     },
2065     base64 => {
2066         type => 'boolean',
2067         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2068         optional => 1,
2069     },
2070 };
2071
2072 sub parse_smbios1 {
2073     my ($data) = @_;
2074
2075     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2076     warn $@ if $@;
2077     return $res;
2078 }
2079
2080 sub print_smbios1 {
2081     my ($smbios1) = @_;
2082     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2083 }
2084
2085 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2086
2087 sub parse_watchdog {
2088     my ($value) = @_;
2089
2090     return if !$value;
2091
2092     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2093     warn $@ if $@;
2094     return $res;
2095 }
2096
2097 sub parse_guest_agent {
2098     my ($conf) = @_;
2099
2100     return {} if !defined($conf->{agent});
2101
2102     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2103     warn $@ if $@;
2104
2105     # if the agent is disabled ignore the other potentially set properties
2106     return {} if !$res->{enabled};
2107     return $res;
2108 }
2109
2110 sub get_qga_key {
2111     my ($conf, $key) = @_;
2112     return undef if !defined($conf->{agent});
2113
2114     my $agent = parse_guest_agent($conf);
2115     return $agent->{$key};
2116 }
2117
2118 sub parse_vga {
2119     my ($value) = @_;
2120
2121     return {} if !$value;
2122     my $res = eval { parse_property_string($vga_fmt, $value) };
2123     warn $@ if $@;
2124     return $res;
2125 }
2126
2127 sub parse_rng {
2128     my ($value) = @_;
2129
2130     return if !$value;
2131
2132     my $res = eval { parse_property_string($rng_fmt, $value) };
2133     warn $@ if $@;
2134     return $res;
2135 }
2136
2137 sub parse_meta_info {
2138     my ($value) = @_;
2139
2140     return if !$value;
2141
2142     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2143     warn $@ if $@;
2144     return $res;
2145 }
2146
2147 sub new_meta_info_string {
2148     my () = @_; # for now do not allow to override any value
2149
2150     return PVE::JSONSchema::print_property_string(
2151         {
2152             'creation-qemu' => kvm_user_version(),
2153             ctime => "". int(time()),
2154         },
2155         $meta_info_fmt
2156     );
2157 }
2158
2159 sub qemu_created_version_fixups {
2160     my ($conf, $forcemachine, $kvmver) = @_;
2161
2162     my $meta = parse_meta_info($conf->{meta}) // {};
2163     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2164
2165     # check if we need to apply some handling for VMs that always use the latest machine version but
2166     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2167     # would be required (we do not want to pin machine version for non-windows OS type)
2168     if (
2169         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2170         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2171         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2172         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2173     ) {
2174         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2175         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2176             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2177             # and thus with the predictable interface naming of systemd
2178             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2179         }
2180     }
2181     return;
2182 }
2183
2184 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2185 sub verify_usb_device {
2186     my ($value, $noerr) = @_;
2187
2188     return $value if parse_usb_device($value);
2189
2190     return if $noerr;
2191
2192     die "unable to parse usb device\n";
2193 }
2194
2195 # add JSON properties for create and set function
2196 sub json_config_properties {
2197     my ($prop, $with_disk_alloc) = @_;
2198
2199     my $skip_json_config_opts = {
2200         parent => 1,
2201         snaptime => 1,
2202         vmstate => 1,
2203         runningmachine => 1,
2204         runningcpu => 1,
2205         meta => 1,
2206     };
2207
2208     foreach my $opt (keys %$confdesc) {
2209         next if $skip_json_config_opts->{$opt};
2210
2211         if ($with_disk_alloc && is_valid_drivename($opt)) {
2212             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2213         } else {
2214             $prop->{$opt} = $confdesc->{$opt};
2215         }
2216     }
2217
2218     return $prop;
2219 }
2220
2221 # Properties that we can read from an OVF file
2222 sub json_ovf_properties {
2223     my $prop = {};
2224
2225     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2226         $prop->{$device} = {
2227             type => 'string',
2228             format => 'pve-volume-id-or-absolute-path',
2229             description => "Disk image that gets imported to $device",
2230             optional => 1,
2231         };
2232     }
2233
2234     $prop->{cores} = {
2235         type => 'integer',
2236         description => "The number of CPU cores.",
2237         optional => 1,
2238     };
2239     $prop->{memory} = {
2240         type => 'integer',
2241         description => "Amount of RAM for the VM in MB.",
2242         optional => 1,
2243     };
2244     $prop->{name} = {
2245         type => 'string',
2246         description => "Name of the VM.",
2247         optional => 1,
2248     };
2249
2250     return $prop;
2251 }
2252
2253 # return copy of $confdesc_cloudinit to generate documentation
2254 sub cloudinit_config_properties {
2255
2256     return dclone($confdesc_cloudinit);
2257 }
2258
2259 sub check_type {
2260     my ($key, $value) = @_;
2261
2262     die "unknown setting '$key'\n" if !$confdesc->{$key};
2263
2264     my $type = $confdesc->{$key}->{type};
2265
2266     if (!defined($value)) {
2267         die "got undefined value\n";
2268     }
2269
2270     if ($value =~ m/[\n\r]/) {
2271         die "property contains a line feed\n";
2272     }
2273
2274     if ($type eq 'boolean') {
2275         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2276         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2277         die "type check ('boolean') failed - got '$value'\n";
2278     } elsif ($type eq 'integer') {
2279         return int($1) if $value =~ m/^(\d+)$/;
2280         die "type check ('integer') failed - got '$value'\n";
2281     } elsif ($type eq 'number') {
2282         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2283         die "type check ('number') failed - got '$value'\n";
2284     } elsif ($type eq 'string') {
2285         if (my $fmt = $confdesc->{$key}->{format}) {
2286             PVE::JSONSchema::check_format($fmt, $value);
2287             return $value;
2288         }
2289         $value =~ s/^\"(.*)\"$/$1/;
2290         return $value;
2291     } else {
2292         die "internal error"
2293     }
2294 }
2295
2296 sub destroy_vm {
2297     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2298
2299     my $conf = PVE::QemuConfig->load_config($vmid);
2300
2301     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2302
2303     if ($conf->{template}) {
2304         # check if any base image is still used by a linked clone
2305         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2306                 my ($ds, $drive) = @_;
2307                 return if drive_is_cdrom($drive);
2308
2309                 my $volid = $drive->{file};
2310                 return if !$volid || $volid =~ m|^/|;
2311
2312                 die "base volume '$volid' is still in use by linked cloned\n"
2313                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2314
2315         });
2316     }
2317
2318     my $volids = {};
2319     my $remove_owned_drive = sub {
2320         my ($ds, $drive) = @_;
2321         return if drive_is_cdrom($drive, 1);
2322
2323         my $volid = $drive->{file};
2324         return if !$volid || $volid =~ m|^/|;
2325         return if $volids->{$volid};
2326
2327         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2328         return if !$path || !$owner || ($owner != $vmid);
2329
2330         $volids->{$volid} = 1;
2331         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2332         warn "Could not remove disk '$volid', check manually: $@" if $@;
2333     };
2334
2335     # only remove disks owned by this VM (referenced in the config)
2336     my $include_opts = {
2337         include_unused => 1,
2338         extra_keys => ['vmstate'],
2339     };
2340     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2341
2342     for my $snap (values %{$conf->{snapshots}}) {
2343         next if !defined($snap->{vmstate});
2344         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2345         next if !defined($drive);
2346         $remove_owned_drive->('vmstate', $drive);
2347     }
2348
2349     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2350
2351     if ($purge_unreferenced) { # also remove unreferenced disk
2352         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2353         PVE::Storage::foreach_volid($vmdisks, sub {
2354             my ($volid, $sid, $volname, $d) = @_;
2355             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2356             warn $@ if $@;
2357         });
2358     }
2359
2360     if (defined $replacement_conf) {
2361         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2362     } else {
2363         PVE::QemuConfig->destroy_config($vmid);
2364     }
2365 }
2366
2367 sub parse_vm_config {
2368     my ($filename, $raw, $strict) = @_;
2369
2370     return if !defined($raw);
2371
2372     my $res = {
2373         digest => Digest::SHA::sha1_hex($raw),
2374         snapshots => {},
2375         pending => {},
2376         cloudinit => {},
2377     };
2378
2379     my $handle_error = sub {
2380         my ($msg) = @_;
2381
2382         if ($strict) {
2383             die $msg;
2384         } else {
2385             warn $msg;
2386         }
2387     };
2388
2389     $filename =~ m|/qemu-server/(\d+)\.conf$|
2390         || die "got strange filename '$filename'";
2391
2392     my $vmid = $1;
2393
2394     my $conf = $res;
2395     my $descr;
2396     my $section = '';
2397
2398     my @lines = split(/\n/, $raw);
2399     foreach my $line (@lines) {
2400         next if $line =~ m/^\s*$/;
2401
2402         if ($line =~ m/^\[PENDING\]\s*$/i) {
2403             $section = 'pending';
2404             if (defined($descr)) {
2405                 $descr =~ s/\s+$//;
2406                 $conf->{description} = $descr;
2407             }
2408             $descr = undef;
2409             $conf = $res->{$section} = {};
2410             next;
2411         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2412             $section = 'cloudinit';
2413             $descr = undef;
2414             $conf = $res->{$section} = {};
2415             next;
2416
2417         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2418             $section = $1;
2419             if (defined($descr)) {
2420                 $descr =~ s/\s+$//;
2421                 $conf->{description} = $descr;
2422             }
2423             $descr = undef;
2424             $conf = $res->{snapshots}->{$section} = {};
2425             next;
2426         }
2427
2428         if ($line =~ m/^\#(.*)$/) {
2429             $descr = '' if !defined($descr);
2430             $descr .= PVE::Tools::decode_text($1) . "\n";
2431             next;
2432         }
2433
2434         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2435             $descr = '' if !defined($descr);
2436             $descr .= PVE::Tools::decode_text($2);
2437         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2438             $conf->{snapstate} = $1;
2439         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2440             my $key = $1;
2441             my $value = $2;
2442             $conf->{$key} = $value;
2443         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2444             my $value = $1;
2445             if ($section eq 'pending') {
2446                 $conf->{delete} = $value; # we parse this later
2447             } else {
2448                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2449             }
2450         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2451             my $key = $1;
2452             my $value = $2;
2453             eval { $value = check_type($key, $value); };
2454             if ($@) {
2455                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2456             } else {
2457                 $key = 'ide2' if $key eq 'cdrom';
2458                 my $fmt = $confdesc->{$key}->{format};
2459                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2460                     my $v = parse_drive($key, $value);
2461                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2462                         $v->{file} = $volid;
2463                         $value = print_drive($v);
2464                     } else {
2465                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2466                         next;
2467                     }
2468                 }
2469
2470                 $conf->{$key} = $value;
2471             }
2472         } else {
2473             $handle_error->("vm $vmid - unable to parse config: $line\n");
2474         }
2475     }
2476
2477     if (defined($descr)) {
2478         $descr =~ s/\s+$//;
2479         $conf->{description} = $descr;
2480     }
2481     delete $res->{snapstate}; # just to be sure
2482
2483     return $res;
2484 }
2485
2486 sub write_vm_config {
2487     my ($filename, $conf) = @_;
2488
2489     delete $conf->{snapstate}; # just to be sure
2490
2491     if ($conf->{cdrom}) {
2492         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2493         $conf->{ide2} = $conf->{cdrom};
2494         delete $conf->{cdrom};
2495     }
2496
2497     # we do not use 'smp' any longer
2498     if ($conf->{sockets}) {
2499         delete $conf->{smp};
2500     } elsif ($conf->{smp}) {
2501         $conf->{sockets} = $conf->{smp};
2502         delete $conf->{cores};
2503         delete $conf->{smp};
2504     }
2505
2506     my $used_volids = {};
2507
2508     my $cleanup_config = sub {
2509         my ($cref, $pending, $snapname) = @_;
2510
2511         foreach my $key (keys %$cref) {
2512             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2513                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2514             my $value = $cref->{$key};
2515             if ($key eq 'delete') {
2516                 die "propertry 'delete' is only allowed in [PENDING]\n"
2517                     if !$pending;
2518                 # fixme: check syntax?
2519                 next;
2520             }
2521             eval { $value = check_type($key, $value); };
2522             die "unable to parse value of '$key' - $@" if $@;
2523
2524             $cref->{$key} = $value;
2525
2526             if (!$snapname && is_valid_drivename($key)) {
2527                 my $drive = parse_drive($key, $value);
2528                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2529             }
2530         }
2531     };
2532
2533     &$cleanup_config($conf);
2534
2535     &$cleanup_config($conf->{pending}, 1);
2536
2537     &$cleanup_config($conf->{cloudinit});
2538
2539     foreach my $snapname (keys %{$conf->{snapshots}}) {
2540         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2541         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2542     }
2543
2544     # remove 'unusedX' settings if we re-add a volume
2545     foreach my $key (keys %$conf) {
2546         my $value = $conf->{$key};
2547         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2548             delete $conf->{$key};
2549         }
2550     }
2551
2552     my $generate_raw_config = sub {
2553         my ($conf, $pending) = @_;
2554
2555         my $raw = '';
2556
2557         # add description as comment to top of file
2558         if (defined(my $descr = $conf->{description})) {
2559             if ($descr) {
2560                 foreach my $cl (split(/\n/, $descr)) {
2561                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2562                 }
2563             } else {
2564                 $raw .= "#\n" if $pending;
2565             }
2566         }
2567
2568         foreach my $key (sort keys %$conf) {
2569             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2570             $raw .= "$key: $conf->{$key}\n";
2571         }
2572         return $raw;
2573     };
2574
2575     my $raw = &$generate_raw_config($conf);
2576
2577     if (scalar(keys %{$conf->{pending}})){
2578         $raw .= "\n[PENDING]\n";
2579         $raw .= &$generate_raw_config($conf->{pending}, 1);
2580     }
2581
2582     if (scalar(keys %{$conf->{cloudinit}})){
2583         $raw .= "\n[special:cloudinit]\n";
2584         $raw .= &$generate_raw_config($conf->{cloudinit});
2585     }
2586
2587     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2588         $raw .= "\n[$snapname]\n";
2589         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2590     }
2591
2592     return $raw;
2593 }
2594
2595 sub load_defaults {
2596
2597     my $res = {};
2598
2599     # we use static defaults from our JSON schema configuration
2600     foreach my $key (keys %$confdesc) {
2601         if (defined(my $default = $confdesc->{$key}->{default})) {
2602             $res->{$key} = $default;
2603         }
2604     }
2605
2606     return $res;
2607 }
2608
2609 sub config_list {
2610     my $vmlist = PVE::Cluster::get_vmlist();
2611     my $res = {};
2612     return $res if !$vmlist || !$vmlist->{ids};
2613     my $ids = $vmlist->{ids};
2614     my $nodename = nodename();
2615
2616     foreach my $vmid (keys %$ids) {
2617         my $d = $ids->{$vmid};
2618         next if !$d->{node} || $d->{node} ne $nodename;
2619         next if !$d->{type} || $d->{type} ne 'qemu';
2620         $res->{$vmid}->{exists} = 1;
2621     }
2622     return $res;
2623 }
2624
2625 # test if VM uses local resources (to prevent migration)
2626 sub check_local_resources {
2627     my ($conf, $noerr) = @_;
2628
2629     my @loc_res = ();
2630
2631     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2632     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2633
2634     push @loc_res, "ivshmem" if $conf->{ivshmem};
2635
2636     foreach my $k (keys %$conf) {
2637         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2638         # sockets are safe: they will recreated be on the target side post-migrate
2639         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2640         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2641     }
2642
2643     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2644
2645     return \@loc_res;
2646 }
2647
2648 # check if used storages are available on all nodes (use by migrate)
2649 sub check_storage_availability {
2650     my ($storecfg, $conf, $node) = @_;
2651
2652     PVE::QemuConfig->foreach_volume($conf, sub {
2653         my ($ds, $drive) = @_;
2654
2655         my $volid = $drive->{file};
2656         return if !$volid;
2657
2658         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2659         return if !$sid;
2660
2661         # check if storage is available on both nodes
2662         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2663         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2664
2665         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2666
2667         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2668             if !$scfg->{content}->{$vtype};
2669    });
2670 }
2671
2672 # list nodes where all VM images are available (used by has_feature API)
2673 sub shared_nodes {
2674     my ($conf, $storecfg) = @_;
2675
2676     my $nodelist = PVE::Cluster::get_nodelist();
2677     my $nodehash = { map { $_ => 1 } @$nodelist };
2678     my $nodename = nodename();
2679
2680     PVE::QemuConfig->foreach_volume($conf, sub {
2681         my ($ds, $drive) = @_;
2682
2683         my $volid = $drive->{file};
2684         return if !$volid;
2685
2686         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2687         if ($storeid) {
2688             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2689             if ($scfg->{disable}) {
2690                 $nodehash = {};
2691             } elsif (my $avail = $scfg->{nodes}) {
2692                 foreach my $node (keys %$nodehash) {
2693                     delete $nodehash->{$node} if !$avail->{$node};
2694                 }
2695             } elsif (!$scfg->{shared}) {
2696                 foreach my $node (keys %$nodehash) {
2697                     delete $nodehash->{$node} if $node ne $nodename
2698                 }
2699             }
2700         }
2701     });
2702
2703     return $nodehash
2704 }
2705
2706 sub check_local_storage_availability {
2707     my ($conf, $storecfg) = @_;
2708
2709     my $nodelist = PVE::Cluster::get_nodelist();
2710     my $nodehash = { map { $_ => {} } @$nodelist };
2711
2712     PVE::QemuConfig->foreach_volume($conf, sub {
2713         my ($ds, $drive) = @_;
2714
2715         my $volid = $drive->{file};
2716         return if !$volid;
2717
2718         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2719         if ($storeid) {
2720             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2721
2722             if ($scfg->{disable}) {
2723                 foreach my $node (keys %$nodehash) {
2724                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2725                 }
2726             } elsif (my $avail = $scfg->{nodes}) {
2727                 foreach my $node (keys %$nodehash) {
2728                     if (!$avail->{$node}) {
2729                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2730                     }
2731                 }
2732             }
2733         }
2734     });
2735
2736     foreach my $node (values %$nodehash) {
2737         if (my $unavail = $node->{unavailable_storages}) {
2738             $node->{unavailable_storages} = [ sort keys %$unavail ];
2739         }
2740     }
2741
2742     return $nodehash
2743 }
2744
2745 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2746 sub check_running {
2747     my ($vmid, $nocheck, $node) = @_;
2748
2749     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2750     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2751 }
2752
2753 sub vzlist {
2754
2755     my $vzlist = config_list();
2756
2757     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2758
2759     while (defined(my $de = $fd->read)) {
2760         next if $de !~ m/^(\d+)\.pid$/;
2761         my $vmid = $1;
2762         next if !defined($vzlist->{$vmid});
2763         if (my $pid = check_running($vmid)) {
2764             $vzlist->{$vmid}->{pid} = $pid;
2765         }
2766     }
2767
2768     return $vzlist;
2769 }
2770
2771 our $vmstatus_return_properties = {
2772     vmid => get_standard_option('pve-vmid'),
2773     status => {
2774         description => "Qemu process status.",
2775         type => 'string',
2776         enum => ['stopped', 'running'],
2777     },
2778     maxmem => {
2779         description => "Maximum memory in bytes.",
2780         type => 'integer',
2781         optional => 1,
2782         renderer => 'bytes',
2783     },
2784     maxdisk => {
2785         description => "Root disk size in bytes.",
2786         type => 'integer',
2787         optional => 1,
2788         renderer => 'bytes',
2789     },
2790     name => {
2791         description => "VM name.",
2792         type => 'string',
2793         optional => 1,
2794     },
2795     qmpstatus => {
2796         description => "Qemu QMP agent status.",
2797         type => 'string',
2798         optional => 1,
2799     },
2800     pid => {
2801         description => "PID of running qemu process.",
2802         type => 'integer',
2803         optional => 1,
2804     },
2805     uptime => {
2806         description => "Uptime.",
2807         type => 'integer',
2808         optional => 1,
2809         renderer => 'duration',
2810     },
2811     cpus => {
2812         description => "Maximum usable CPUs.",
2813         type => 'number',
2814         optional => 1,
2815     },
2816     lock => {
2817         description => "The current config lock, if any.",
2818         type => 'string',
2819         optional => 1,
2820     },
2821     tags => {
2822         description  => "The current configured tags, if any",
2823         type => 'string',
2824         optional => 1,
2825     },
2826     'running-machine' => {
2827         description => "The currently running machine type (if running).",
2828         type => 'string',
2829         optional => 1,
2830     },
2831     'running-qemu' => {
2832         description => "The currently running QEMU version (if running).",
2833         type => 'string',
2834         optional => 1,
2835     },
2836 };
2837
2838 my $last_proc_pid_stat;
2839
2840 # get VM status information
2841 # This must be fast and should not block ($full == false)
2842 # We only query KVM using QMP if $full == true (this can be slow)
2843 sub vmstatus {
2844     my ($opt_vmid, $full) = @_;
2845
2846     my $res = {};
2847
2848     my $storecfg = PVE::Storage::config();
2849
2850     my $list = vzlist();
2851     my $defaults = load_defaults();
2852
2853     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2854
2855     my $cpucount = $cpuinfo->{cpus} || 1;
2856
2857     foreach my $vmid (keys %$list) {
2858         next if $opt_vmid && ($vmid ne $opt_vmid);
2859
2860         my $conf = PVE::QemuConfig->load_config($vmid);
2861
2862         my $d = { vmid => int($vmid) };
2863         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2864
2865         # fixme: better status?
2866         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2867
2868         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2869         if (defined($size)) {
2870             $d->{disk} = 0; # no info available
2871             $d->{maxdisk} = $size;
2872         } else {
2873             $d->{disk} = 0;
2874             $d->{maxdisk} = 0;
2875         }
2876
2877         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2878             * ($conf->{cores} || $defaults->{cores});
2879         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2880         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2881
2882         $d->{name} = $conf->{name} || "VM $vmid";
2883         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2884             : $defaults->{memory}*(1024*1024);
2885
2886         if ($conf->{balloon}) {
2887             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2888             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2889                 : $defaults->{shares};
2890         }
2891
2892         $d->{uptime} = 0;
2893         $d->{cpu} = 0;
2894         $d->{mem} = 0;
2895
2896         $d->{netout} = 0;
2897         $d->{netin} = 0;
2898
2899         $d->{diskread} = 0;
2900         $d->{diskwrite} = 0;
2901
2902         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2903
2904         $d->{serial} = 1 if conf_has_serial($conf);
2905         $d->{lock} = $conf->{lock} if $conf->{lock};
2906         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2907
2908         $res->{$vmid} = $d;
2909     }
2910
2911     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2912     foreach my $dev (keys %$netdev) {
2913         next if $dev !~ m/^tap([1-9]\d*)i/;
2914         my $vmid = $1;
2915         my $d = $res->{$vmid};
2916         next if !$d;
2917
2918         $d->{netout} += $netdev->{$dev}->{receive};
2919         $d->{netin} += $netdev->{$dev}->{transmit};
2920
2921         if ($full) {
2922             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2923             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2924         }
2925
2926     }
2927
2928     my $ctime = gettimeofday;
2929
2930     foreach my $vmid (keys %$list) {
2931
2932         my $d = $res->{$vmid};
2933         my $pid = $d->{pid};
2934         next if !$pid;
2935
2936         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2937         next if !$pstat; # not running
2938
2939         my $used = $pstat->{utime} + $pstat->{stime};
2940
2941         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2942
2943         if ($pstat->{vsize}) {
2944             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2945         }
2946
2947         my $old = $last_proc_pid_stat->{$pid};
2948         if (!$old) {
2949             $last_proc_pid_stat->{$pid} = {
2950                 time => $ctime,
2951                 used => $used,
2952                 cpu => 0,
2953             };
2954             next;
2955         }
2956
2957         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2958
2959         if ($dtime > 1000) {
2960             my $dutime = $used -  $old->{used};
2961
2962             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2963             $last_proc_pid_stat->{$pid} = {
2964                 time => $ctime,
2965                 used => $used,
2966                 cpu => $d->{cpu},
2967             };
2968         } else {
2969             $d->{cpu} = $old->{cpu};
2970         }
2971     }
2972
2973     return $res if !$full;
2974
2975     my $qmpclient = PVE::QMPClient->new();
2976
2977     my $ballooncb = sub {
2978         my ($vmid, $resp) = @_;
2979
2980         my $info = $resp->{'return'};
2981         return if !$info->{max_mem};
2982
2983         my $d = $res->{$vmid};
2984
2985         # use memory assigned to VM
2986         $d->{maxmem} = $info->{max_mem};
2987         $d->{balloon} = $info->{actual};
2988
2989         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2990             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2991             $d->{freemem} = $info->{free_mem};
2992         }
2993
2994         $d->{ballooninfo} = $info;
2995     };
2996
2997     my $blockstatscb = sub {
2998         my ($vmid, $resp) = @_;
2999         my $data = $resp->{'return'} || [];
3000         my $totalrdbytes = 0;
3001         my $totalwrbytes = 0;
3002
3003         for my $blockstat (@$data) {
3004             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3005             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3006
3007             $blockstat->{device} =~ s/drive-//;
3008             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3009         }
3010         $res->{$vmid}->{diskread} = $totalrdbytes;
3011         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3012     };
3013
3014     my $machinecb = sub {
3015         my ($vmid, $resp) = @_;
3016         my $data = $resp->{'return'} || [];
3017
3018         $res->{$vmid}->{'running-machine'} =
3019             PVE::QemuServer::Machine::current_from_query_machines($data);
3020     };
3021
3022     my $versioncb = sub {
3023         my ($vmid, $resp) = @_;
3024         my $data = $resp->{'return'} // {};
3025         my $version = 'unknown';
3026
3027         if (my $v = $data->{qemu}) {
3028             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3029         }
3030
3031         $res->{$vmid}->{'running-qemu'} = $version;
3032     };
3033
3034     my $statuscb = sub {
3035         my ($vmid, $resp) = @_;
3036
3037         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3038         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3039         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3040         # this fails if ballon driver is not loaded, so this must be
3041         # the last commnand (following command are aborted if this fails).
3042         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3043
3044         my $status = 'unknown';
3045         if (!defined($status = $resp->{'return'}->{status})) {
3046             warn "unable to get VM status\n";
3047             return;
3048         }
3049
3050         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3051     };
3052
3053     foreach my $vmid (keys %$list) {
3054         next if $opt_vmid && ($vmid ne $opt_vmid);
3055         next if !$res->{$vmid}->{pid}; # not running
3056         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3057     }
3058
3059     $qmpclient->queue_execute(undef, 2);
3060
3061     foreach my $vmid (keys %$list) {
3062         next if $opt_vmid && ($vmid ne $opt_vmid);
3063         next if !$res->{$vmid}->{pid}; #not running
3064
3065         # we can't use the $qmpclient since it might have already aborted on
3066         # 'query-balloon', but this might also fail for older versions...
3067         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3068         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3069     }
3070
3071     foreach my $vmid (keys %$list) {
3072         next if $opt_vmid && ($vmid ne $opt_vmid);
3073         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3074     }
3075
3076     return $res;
3077 }
3078
3079 sub conf_has_serial {
3080     my ($conf) = @_;
3081
3082     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3083         if ($conf->{"serial$i"}) {
3084             return 1;
3085         }
3086     }
3087
3088     return 0;
3089 }
3090
3091 sub conf_has_audio {
3092     my ($conf, $id) = @_;
3093
3094     $id //= 0;
3095     my $audio = $conf->{"audio$id"};
3096     return if !defined($audio);
3097
3098     my $audioproperties = parse_property_string($audio_fmt, $audio);
3099     my $audiodriver = $audioproperties->{driver} // 'spice';
3100
3101     return {
3102         dev => $audioproperties->{device},
3103         dev_id => "audiodev$id",
3104         backend => $audiodriver,
3105         backend_id => "$audiodriver-backend${id}",
3106     };
3107 }
3108
3109 sub audio_devs {
3110     my ($audio, $audiopciaddr, $machine_version) = @_;
3111
3112     my $devs = [];
3113
3114     my $id = $audio->{dev_id};
3115     my $audiodev = "";
3116     if (min_version($machine_version, 4, 2)) {
3117         $audiodev = ",audiodev=$audio->{backend_id}";
3118     }
3119
3120     if ($audio->{dev} eq 'AC97') {
3121         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3122     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3123         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3124         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3125         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3126     } else {
3127         die "unkown audio device '$audio->{dev}', implement me!";
3128     }
3129
3130     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3131
3132     return $devs;
3133 }
3134
3135 sub get_tpm_paths {
3136     my ($vmid) = @_;
3137     return {
3138         socket => "/var/run/qemu-server/$vmid.swtpm",
3139         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3140     };
3141 }
3142
3143 sub add_tpm_device {
3144     my ($vmid, $devices, $conf) = @_;
3145
3146     return if !$conf->{tpmstate0};
3147
3148     my $paths = get_tpm_paths($vmid);
3149
3150     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3151     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3152     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3153 }
3154
3155 sub start_swtpm {
3156     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3157
3158     return if !$tpmdrive;
3159
3160     my $state;
3161     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3162     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3163     if ($storeid) {
3164         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3165     } else {
3166         $state = $tpm->{file};
3167     }
3168
3169     my $paths = get_tpm_paths($vmid);
3170
3171     # during migration, we will get state from remote
3172     #
3173     if (!$migration) {
3174         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3175         my $setup_cmd = [
3176             "swtpm_setup",
3177             "--tpmstate",
3178             "file://$state",
3179             "--createek",
3180             "--create-ek-cert",
3181             "--create-platform-cert",
3182             "--lock-nvram",
3183             "--config",
3184             "/etc/swtpm_setup.conf", # do not use XDG configs
3185             "--runas",
3186             "0", # force creation as root, error if not possible
3187             "--not-overwrite", # ignore existing state, do not modify
3188         ];
3189
3190         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3191         # TPM 2.0 supports ECC crypto, use if possible
3192         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3193
3194         run_command($setup_cmd, outfunc => sub {
3195             print "swtpm_setup: $1\n";
3196         });
3197     }
3198
3199     my $emulator_cmd = [
3200         "swtpm",
3201         "socket",
3202         "--tpmstate",
3203         "backend-uri=file://$state,mode=0600",
3204         "--ctrl",
3205         "type=unixio,path=$paths->{socket},mode=0600",
3206         "--pid",
3207         "file=$paths->{pid}",
3208         "--terminate", # terminate on QEMU disconnect
3209         "--daemon",
3210     ];
3211     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3212     run_command($emulator_cmd, outfunc => sub { print $1; });
3213
3214     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3215     while (! -e $paths->{pid}) {
3216         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3217         usleep(50_000);
3218     }
3219
3220     # return untainted PID of swtpm daemon so it can be killed on error
3221     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3222     return $1;
3223 }
3224
3225 sub vga_conf_has_spice {
3226     my ($vga) = @_;
3227
3228     my $vgaconf = parse_vga($vga);
3229     my $vgatype = $vgaconf->{type};
3230     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3231
3232     return $1 || 1;
3233 }
3234
3235 sub is_native($) {
3236     my ($arch) = @_;
3237     return get_host_arch() eq $arch;
3238 }
3239
3240 sub get_vm_arch {
3241     my ($conf) = @_;
3242     return $conf->{arch} // get_host_arch();
3243 }
3244
3245 my $default_machines = {
3246     x86_64 => 'pc',
3247     aarch64 => 'virt',
3248 };
3249
3250 sub get_installed_machine_version {
3251     my ($kvmversion) = @_;
3252     $kvmversion = kvm_user_version() if !defined($kvmversion);
3253     $kvmversion =~ m/^(\d+\.\d+)/;
3254     return $1;
3255 }
3256
3257 sub windows_get_pinned_machine_version {
3258     my ($machine, $base_version, $kvmversion) = @_;
3259
3260     my $pin_version = $base_version;
3261     if (!defined($base_version) ||
3262         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3263     ) {
3264         $pin_version = get_installed_machine_version($kvmversion);
3265     }
3266     if (!$machine || $machine eq 'pc') {
3267         $machine = "pc-i440fx-$pin_version";
3268     } elsif ($machine eq 'q35') {
3269         $machine = "pc-q35-$pin_version";
3270     } elsif ($machine eq 'virt') {
3271         $machine = "virt-$pin_version";
3272     } else {
3273         warn "unknown machine type '$machine', not touching that!\n";
3274     }
3275
3276     return $machine;
3277 }
3278
3279 sub get_vm_machine {
3280     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3281
3282     my $machine = $forcemachine || $conf->{machine};
3283
3284     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3285         $kvmversion //= kvm_user_version();
3286         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3287         # layout which confuses windows quite a bit and may result in various regressions..
3288         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3289         if (windows_version($conf->{ostype})) {
3290             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3291         }
3292         $arch //= 'x86_64';
3293         $machine ||= $default_machines->{$arch};
3294         if ($add_pve_version) {
3295             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3296             $machine .= "+pve$pvever";
3297         }
3298     }
3299
3300     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3301         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3302         $machine = $1 if $is_pxe;
3303
3304         # for version-pinned machines that do not include a pve-version (e.g.
3305         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3306         $machine .= '+pve0';
3307
3308         $machine .= '.pxe' if $is_pxe;
3309     }
3310
3311     return $machine;
3312 }
3313
3314 sub get_ovmf_files($$$) {
3315     my ($arch, $efidisk, $smm) = @_;
3316
3317     my $types = $OVMF->{$arch}
3318         or die "no OVMF images known for architecture '$arch'\n";
3319
3320     my $type = 'default';
3321     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3322         $type = $smm ? "4m" : "4m-no-smm";
3323         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3324     }
3325
3326     return $types->{$type}->@*;
3327 }
3328
3329 my $Arch2Qemu = {
3330     aarch64 => '/usr/bin/qemu-system-aarch64',
3331     x86_64 => '/usr/bin/qemu-system-x86_64',
3332 };
3333 sub get_command_for_arch($) {
3334     my ($arch) = @_;
3335     return '/usr/bin/kvm' if is_native($arch);
3336
3337     my $cmd = $Arch2Qemu->{$arch}
3338         or die "don't know how to emulate architecture '$arch'\n";
3339     return $cmd;
3340 }
3341
3342 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3343 # to use in a QEMU command line (-cpu element), first array_intersect the result
3344 # of query_supported_ with query_understood_. This is necessary because:
3345 #
3346 # a) query_understood_ returns flags the host cannot use and
3347 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3348 #    flags, but CPU settings - with most of them being flags. Those settings
3349 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3350 #
3351 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3352 # expensive. If you need the value returned from this, you can get it much
3353 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3354 # $accel being 'kvm' or 'tcg'.
3355 #
3356 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3357 # changes, automatically populating pmxcfs.
3358 #
3359 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3360 # since kvm and tcg machines support different flags
3361 #
3362 sub query_supported_cpu_flags {
3363     my ($arch) = @_;
3364
3365     $arch //= get_host_arch();
3366     my $default_machine = $default_machines->{$arch};
3367
3368     my $flags = {};
3369
3370     # FIXME: Once this is merged, the code below should work for ARM as well:
3371     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3372     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3373         $arch eq "aarch64";
3374
3375     my $kvm_supported = defined(kvm_version());
3376     my $qemu_cmd = get_command_for_arch($arch);
3377     my $fakevmid = -1;
3378     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3379
3380     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3381     my $query_supported_run_qemu = sub {
3382         my ($kvm) = @_;
3383
3384         my $flags = {};
3385         my $cmd = [
3386             $qemu_cmd,
3387             '-machine', $default_machine,
3388             '-display', 'none',
3389             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3390             '-mon', 'chardev=qmp,mode=control',
3391             '-pidfile', $pidfile,
3392             '-S', '-daemonize'
3393         ];
3394
3395         if (!$kvm) {
3396             push @$cmd, '-accel', 'tcg';
3397         }
3398
3399         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3400         die "QEMU flag querying VM exited with code " . $rc if $rc;
3401
3402         eval {
3403             my $cmd_result = mon_cmd(
3404                 $fakevmid,
3405                 'query-cpu-model-expansion',
3406                 type => 'full',
3407                 model => { name => 'host' }
3408             );
3409
3410             my $props = $cmd_result->{model}->{props};
3411             foreach my $prop (keys %$props) {
3412                 next if $props->{$prop} ne '1';
3413                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3414                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3415                 # We only keep those with underscores, to match /proc/cpuinfo
3416                 $prop =~ s/\.|-/_/g;
3417                 $flags->{$prop} = 1;
3418             }
3419         };
3420         my $err = $@;
3421
3422         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3423         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3424
3425         die $err if $err;
3426
3427         return [ sort keys %$flags ];
3428     };
3429
3430     # We need to query QEMU twice, since KVM and TCG have different supported flags
3431     PVE::QemuConfig->lock_config($fakevmid, sub {
3432         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3433         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3434
3435         if ($kvm_supported) {
3436             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3437             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3438         }
3439     });
3440
3441     return $flags;
3442 }
3443
3444 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3445 my $understood_cpu_flag_dir = "/usr/share/kvm";
3446 sub query_understood_cpu_flags {
3447     my $arch = get_host_arch();
3448     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3449
3450     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3451         if ! -e $filepath;
3452
3453     my $raw = file_get_contents($filepath);
3454     $raw =~ s/^\s+|\s+$//g;
3455     my @flags = split(/\s+/, $raw);
3456
3457     return \@flags;
3458 }
3459
3460 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3461 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3462 my sub should_disable_smm {
3463     my ($conf, $vga) = @_;
3464
3465     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3466         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3467 }
3468
3469 sub config_to_command {
3470     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3471         $pbs_backing) = @_;
3472
3473     my $cmd = [];
3474     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3475     my $devices = [];
3476     my $bridges = {};
3477     my $ostype = $conf->{ostype};
3478     my $winversion = windows_version($ostype);
3479     my $kvm = $conf->{kvm};
3480     my $nodename = nodename();
3481
3482     my $arch = get_vm_arch($conf);
3483     my $kvm_binary = get_command_for_arch($arch);
3484     my $kvmver = kvm_user_version($kvm_binary);
3485
3486     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3487         $kvmver //= "undefined";
3488         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3489     }
3490
3491     my $add_pve_version = min_version($kvmver, 4, 1);
3492
3493     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3494     my $machine_version = extract_version($machine_type, $kvmver);
3495     $kvm //= 1 if is_native($arch);
3496
3497     $machine_version =~ m/(\d+)\.(\d+)/;
3498     my ($machine_major, $machine_minor) = ($1, $2);
3499
3500     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3501         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3502     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3503         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3504             ." please upgrade node '$nodename'\n"
3505     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3506         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3507         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3508             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3509             ." node '$nodename'\n";
3510     }
3511
3512     # if a specific +pve version is required for a feature, use $version_guard
3513     # instead of min_version to allow machines to be run with the minimum
3514     # required version
3515     my $required_pve_version = 0;
3516     my $version_guard = sub {
3517         my ($major, $minor, $pve) = @_;
3518         return 0 if !min_version($machine_version, $major, $minor, $pve);
3519         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3520         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3521         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3522         return 1;
3523     };
3524
3525     if ($kvm && !defined kvm_version()) {
3526         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3527             ." or enable in BIOS.\n";
3528     }
3529
3530     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3531     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3532     my $use_old_bios_files = undef;
3533     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3534
3535     push @$cmd, $kvm_binary;
3536
3537     push @$cmd, '-id', $vmid;
3538
3539     my $vmname = $conf->{name} || "vm$vmid";
3540
3541     push @$cmd, '-name', "$vmname,debug-threads=on";
3542
3543     push @$cmd, '-no-shutdown';
3544
3545     my $use_virtio = 0;
3546
3547     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3548     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3549     push @$cmd, '-mon', "chardev=qmp,mode=control";
3550
3551     if (min_version($machine_version, 2, 12)) {
3552         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3553         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3554     }
3555
3556     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3557
3558     push @$cmd, '-daemonize';
3559
3560     if ($conf->{smbios1}) {
3561         my $smbios_conf = parse_smbios1($conf->{smbios1});
3562         if ($smbios_conf->{base64}) {
3563             # Do not pass base64 flag to qemu
3564             delete $smbios_conf->{base64};
3565             my $smbios_string = "";
3566             foreach my $key (keys %$smbios_conf) {
3567                 my $value;
3568                 if ($key eq "uuid") {
3569                     $value = $smbios_conf->{uuid}
3570                 } else {
3571                     $value = decode_base64($smbios_conf->{$key});
3572                 }
3573                 # qemu accepts any binary data, only commas need escaping by double comma
3574                 $value =~ s/,/,,/g;
3575                 $smbios_string .= "," . $key . "=" . $value if $value;
3576             }
3577             push @$cmd, '-smbios', "type=1" . $smbios_string;
3578         } else {
3579             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3580         }
3581     }
3582
3583     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3584         my $d;
3585         if (my $efidisk = $conf->{efidisk0}) {
3586             $d = parse_drive('efidisk0', $efidisk);
3587         }
3588
3589         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3590         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3591
3592         my ($path, $format);
3593         my $read_only_str = '';
3594         if ($d) {
3595             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3596             $format = $d->{format};
3597             if ($storeid) {
3598                 $path = PVE::Storage::path($storecfg, $d->{file});
3599                 if (!defined($format)) {
3600                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3601                     $format = qemu_img_format($scfg, $volname);
3602                 }
3603             } else {
3604                 $path = $d->{file};
3605                 die "efidisk format must be specified\n"
3606                     if !defined($format);
3607             }
3608
3609             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3610         } else {
3611             log_warn("no efidisk configured! Using temporary efivars disk.");
3612             $path = "/tmp/$vmid-ovmf.fd";
3613             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3614             $format = 'raw';
3615         }
3616
3617         my $size_str = "";
3618
3619         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3620             $size_str = ",size=" . (-s $ovmf_vars);
3621         }
3622
3623         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3624         my $cache = "";
3625         if ($path =~ m/^rbd:/) {
3626                 $cache = ',cache=writeback';
3627                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3628         }
3629
3630         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3631         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3632     }
3633
3634     if ($q35) { # tell QEMU to load q35 config early
3635         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3636         if (min_version($machine_version, 4, 0)) {
3637             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3638         } else {
3639             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3640         }
3641     }
3642
3643     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3644         push @$cmd, $fixups->@*;
3645     }
3646
3647     if ($conf->{vmgenid}) {
3648         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3649     }
3650
3651     # add usb controllers
3652     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3653         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3654     push @$devices, @usbcontrollers if @usbcontrollers;
3655     my $vga = parse_vga($conf->{vga});
3656
3657     my $qxlnum = vga_conf_has_spice($conf->{vga});
3658     $vga->{type} = 'qxl' if $qxlnum;
3659
3660     if (!$vga->{type}) {
3661         if ($arch eq 'aarch64') {
3662             $vga->{type} = 'virtio';
3663         } elsif (min_version($machine_version, 2, 9)) {
3664             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3665         } else {
3666             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3667         }
3668     }
3669
3670     # enable absolute mouse coordinates (needed by vnc)
3671     my $tablet = $conf->{tablet};
3672     if (!defined($tablet)) {
3673         $tablet = $defaults->{tablet};
3674         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3675         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3676     }
3677
3678     if ($tablet) {
3679         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3680         my $kbd = print_keyboarddevice_full($conf, $arch);
3681         push @$devices, '-device', $kbd if defined($kbd);
3682     }
3683
3684     my $bootorder = device_bootorder($conf);
3685
3686     # host pci device passthrough
3687     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3688         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3689
3690     # usb devices
3691     my $usb_dev_features = {};
3692     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3693
3694     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3695         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3696     push @$devices, @usbdevices if @usbdevices;
3697
3698     # serial devices
3699     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3700         my $path = $conf->{"serial$i"} or next;
3701         if ($path eq 'socket') {
3702             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3703             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3704             # On aarch64, serial0 is the UART device. Qemu only allows
3705             # connecting UART devices via the '-serial' command line, as
3706             # the device has a fixed slot on the hardware...
3707             if ($arch eq 'aarch64' && $i == 0) {
3708                 push @$devices, '-serial', "chardev:serial$i";
3709             } else {
3710                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3711             }
3712         } else {
3713             die "no such serial device\n" if ! -c $path;
3714             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3715             push @$devices, '-device', "isa-serial,chardev=serial$i";
3716         }
3717     }
3718
3719     # parallel devices
3720     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3721         if (my $path = $conf->{"parallel$i"}) {
3722             die "no such parallel device\n" if ! -c $path;
3723             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3724             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3725             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3726         }
3727     }
3728
3729     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3730         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3731         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3732         push @$devices, @$audio_devs;
3733     }
3734
3735     add_tpm_device($vmid, $devices, $conf);
3736
3737     my $sockets = 1;
3738     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3739     $sockets = $conf->{sockets} if  $conf->{sockets};
3740
3741     my $cores = $conf->{cores} || 1;
3742
3743     my $maxcpus = $sockets * $cores;
3744
3745     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3746
3747     my $allowed_vcpus = $cpuinfo->{cpus};
3748
3749     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3750
3751     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3752         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753         for (my $i = 2; $i <= $vcpus; $i++)  {
3754             my $cpustr = print_cpu_device($conf,$i);
3755             push @$cmd, '-device', $cpustr;
3756         }
3757
3758     } else {
3759
3760         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3761     }
3762     push @$cmd, '-nodefaults';
3763
3764     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3765
3766     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3767
3768     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3769
3770     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3771         push @$devices, '-device', print_vga_device(
3772             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3773
3774         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3775
3776         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3777         push @$cmd,  '-vnc', "unix:$socket,password=on";
3778     } else {
3779         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3780         push @$cmd, '-nographic';
3781     }
3782
3783     # time drift fix
3784     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3785     my $useLocaltime = $conf->{localtime};
3786
3787     if ($winversion >= 5) { # windows
3788         $useLocaltime = 1 if !defined($conf->{localtime});
3789
3790         # use time drift fix when acpi is enabled
3791         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3792             $tdf = 1 if !defined($conf->{tdf});
3793         }
3794     }
3795
3796     if ($winversion >= 6) {
3797         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3798         push @$cmd, '-no-hpet';
3799     }
3800
3801     push @$rtcFlags, 'driftfix=slew' if $tdf;
3802
3803     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3804         push @$rtcFlags, "base=$conf->{startdate}";
3805     } elsif ($useLocaltime) {
3806         push @$rtcFlags, 'base=localtime';
3807     }
3808
3809     if ($forcecpu) {
3810         push @$cmd, '-cpu', $forcecpu;
3811     } else {
3812         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3813     }
3814
3815     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3816
3817     push @$cmd, '-S' if $conf->{freeze};
3818
3819     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3820
3821     my $guest_agent = parse_guest_agent($conf);
3822
3823     if ($guest_agent->{enabled}) {
3824         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3825         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3826
3827         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3828             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3829             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3830             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3831         } elsif ($guest_agent->{type} eq 'isa') {
3832             push @$devices, '-device', "isa-serial,chardev=qga0";
3833         }
3834     }
3835
3836     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3837     if ($rng && $version_guard->(4, 1, 2)) {
3838         check_rng_source($rng->{source});
3839
3840         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3841         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3842         my $limiter_str = "";
3843         if ($max_bytes) {
3844             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3845         }
3846
3847         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3848         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3849         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3850     }
3851
3852     my $spice_port;
3853
3854     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3855         if ($qxlnum > 1) {
3856             if ($winversion){
3857                 for (my $i = 1; $i < $qxlnum; $i++){
3858                     push @$devices, '-device', print_vga_device(
3859                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3860                 }
3861             } else {
3862                 # assume other OS works like Linux
3863                 my ($ram, $vram) = ("134217728", "67108864");
3864                 if ($vga->{memory}) {
3865                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3866                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3867                 }
3868                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3869                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3870             }
3871         }
3872
3873         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3874
3875         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3876         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3877         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3878
3879         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3880         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3881         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3882
3883         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3884         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3885
3886         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3887         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3888         if ($spice_enhancement->{foldersharing}) {
3889             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3890             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3891         }
3892
3893         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3894         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3895             if $spice_enhancement->{videostreaming};
3896
3897         push @$devices, '-spice', "$spice_opts";
3898     }
3899
3900     # enable balloon by default, unless explicitly disabled
3901     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3902         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3903         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3904         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3905         push @$devices, '-device', $ballooncmd;
3906     }
3907
3908     if ($conf->{watchdog}) {
3909         my $wdopts = parse_watchdog($conf->{watchdog});
3910         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3911         my $watchdog = $wdopts->{model} || 'i6300esb';
3912         push @$devices, '-device', "$watchdog$pciaddr";
3913         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3914     }
3915
3916     my $vollist = [];
3917     my $scsicontroller = {};
3918     my $ahcicontroller = {};
3919     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3920
3921     # Add iscsi initiator name if available
3922     if (my $initiator = get_initiator_name()) {
3923         push @$devices, '-iscsi', "initiator-name=$initiator";
3924     }
3925
3926     PVE::QemuConfig->foreach_volume($conf, sub {
3927         my ($ds, $drive) = @_;
3928
3929         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3930             check_volume_storage_type($storecfg, $drive->{file});
3931             push @$vollist, $drive->{file};
3932         }
3933
3934         # ignore efidisk here, already added in bios/fw handling code above
3935         return if $drive->{interface} eq 'efidisk';
3936         # similar for TPM
3937         return if $drive->{interface} eq 'tpmstate';
3938
3939         $use_virtio = 1 if $ds =~ m/^virtio/;
3940
3941         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3942
3943         if ($drive->{interface} eq 'virtio'){
3944            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3945         }
3946
3947         if ($drive->{interface} eq 'scsi') {
3948
3949             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3950
3951             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3952                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3953
3954             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3955             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3956
3957             my $iothread = '';
3958             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3959                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3960                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3961             } elsif ($drive->{iothread}) {
3962                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3963             }
3964
3965             my $queues = '';
3966             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3967                 $queues = ",num_queues=$drive->{queues}";
3968             }
3969
3970             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3971                 if !$scsicontroller->{$controller};
3972             $scsicontroller->{$controller}=1;
3973         }
3974
3975         if ($drive->{interface} eq 'sata') {
3976             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3977             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3978             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3979                 if !$ahcicontroller->{$controller};
3980             $ahcicontroller->{$controller}=1;
3981         }
3982
3983         my $pbs_conf = $pbs_backing->{$ds};
3984         my $pbs_name = undef;
3985         if ($pbs_conf) {
3986             $pbs_name = "drive-$ds-pbs";
3987             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3988         }
3989
3990         my $drive_cmd = print_drive_commandline_full(
3991             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3992
3993         # extra protection for templates, but SATA and IDE don't support it..
3994         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3995
3996         push @$devices, '-drive',$drive_cmd;
3997         push @$devices, '-device', print_drivedevice_full(
3998             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3999     });
4000
4001     for (my $i = 0; $i < $MAX_NETS; $i++) {
4002         my $netname = "net$i";
4003
4004         next if !$conf->{$netname};
4005         my $d = parse_net($conf->{$netname});
4006         next if !$d;
4007
4008         $use_virtio = 1 if $d->{model} eq 'virtio';
4009
4010         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4011
4012         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4013         push @$devices, '-netdev', $netdevfull;
4014
4015         my $netdevicefull = print_netdevice_full(
4016             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4017
4018         push @$devices, '-device', $netdevicefull;
4019     }
4020
4021     if ($conf->{ivshmem}) {
4022         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4023
4024         my $bus;
4025         if ($q35) {
4026             $bus = print_pcie_addr("ivshmem");
4027         } else {
4028             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4029         }
4030
4031         my $ivshmem_name = $ivshmem->{name} // $vmid;
4032         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4033
4034         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4035         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4036             .",size=$ivshmem->{size}M";
4037     }
4038
4039     # pci.4 is nested in pci.1
4040     $bridges->{1} = 1 if $bridges->{4};
4041
4042     if (!$q35) { # add pci bridges
4043         if (min_version($machine_version, 2, 3)) {
4044            $bridges->{1} = 1;
4045            $bridges->{2} = 1;
4046         }
4047         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4048     }
4049
4050     for my $k (sort {$b cmp $a} keys %$bridges) {
4051         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4052
4053         my $k_name = $k;
4054         if ($k == 2 && $legacy_igd) {
4055             $k_name = "$k-igd";
4056         }
4057         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4058         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4059
4060         if ($q35) { # add after -readconfig pve-q35.cfg
4061             splice @$devices, 2, 0, '-device', $devstr;
4062         } else {
4063             unshift @$devices, '-device', $devstr if $k > 0;
4064         }
4065     }
4066
4067     if (!$kvm) {
4068         push @$machineFlags, 'accel=tcg';
4069     }
4070
4071     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4072
4073     my $machine_type_min = $machine_type;
4074     if ($add_pve_version) {
4075         $machine_type_min =~ s/\+pve\d+$//;
4076         $machine_type_min .= "+pve$required_pve_version";
4077     }
4078     push @$machineFlags, "type=${machine_type_min}";
4079
4080     push @$cmd, @$devices;
4081     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4082     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4083     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4084
4085     if (my $vmstate = $conf->{vmstate}) {
4086         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4087         push @$vollist, $vmstate;
4088         push @$cmd, '-loadstate', $statepath;
4089         print "activating and using '$vmstate' as vmstate\n";
4090     }
4091
4092     if (PVE::QemuConfig->is_template($conf)) {
4093         # needed to workaround base volumes being read-only
4094         push @$cmd, '-snapshot';
4095     }
4096
4097     # add custom args
4098     if ($conf->{args}) {
4099         my $aa = PVE::Tools::split_args($conf->{args});
4100         push @$cmd, @$aa;
4101     }
4102
4103     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4104 }
4105
4106 sub check_rng_source {
4107     my ($source) = @_;
4108
4109     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4110     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4111         if ! -e $source;
4112
4113     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4114     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4115         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4116         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4117         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4118             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4119             ." to the host.\n";
4120     }
4121 }
4122
4123 sub spice_port {
4124     my ($vmid) = @_;
4125
4126     my $res = mon_cmd($vmid, 'query-spice');
4127
4128     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4129 }
4130
4131 sub vm_devices_list {
4132     my ($vmid) = @_;
4133
4134     my $res = mon_cmd($vmid, 'query-pci');
4135     my $devices_to_check = [];
4136     my $devices = {};
4137     foreach my $pcibus (@$res) {
4138         push @$devices_to_check, @{$pcibus->{devices}},
4139     }
4140
4141     while (@$devices_to_check) {
4142         my $to_check = [];
4143         for my $d (@$devices_to_check) {
4144             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4145             next if !$d->{'pci_bridge'};
4146
4147             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4148             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4149         }
4150         $devices_to_check = $to_check;
4151     }
4152
4153     my $resblock = mon_cmd($vmid, 'query-block');
4154     foreach my $block (@$resblock) {
4155         if($block->{device} =~ m/^drive-(\S+)/){
4156                 $devices->{$1} = 1;
4157         }
4158     }
4159
4160     my $resmice = mon_cmd($vmid, 'query-mice');
4161     foreach my $mice (@$resmice) {
4162         if ($mice->{name} eq 'QEMU HID Tablet') {
4163             $devices->{tablet} = 1;
4164             last;
4165         }
4166     }
4167
4168     # for usb devices there is no query-usb
4169     # but we can iterate over the entries in
4170     # qom-list path=/machine/peripheral
4171     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4172     foreach my $per (@$resperipheral) {
4173         if ($per->{name} =~ m/^usb\d+$/) {
4174             $devices->{$per->{name}} = 1;
4175         }
4176     }
4177
4178     return $devices;
4179 }
4180
4181 sub vm_deviceplug {
4182     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4183
4184     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4185
4186     my $devices_list = vm_devices_list($vmid);
4187     return 1 if defined($devices_list->{$deviceid});
4188
4189     # add PCI bridge if we need it for the device
4190     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4191
4192     if ($deviceid eq 'tablet') {
4193         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4194     } elsif ($deviceid eq 'keyboard') {
4195         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4196     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4197         die "usb hotplug currently not reliable\n";
4198         # since we can't reliably hot unplug all added usb devices and usb
4199         # passthrough breaks live migration we disable usb hotplugging for now
4200         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4201     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4202         qemu_iothread_add($vmid, $deviceid, $device);
4203
4204         qemu_driveadd($storecfg, $vmid, $device);
4205         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4206
4207         qemu_deviceadd($vmid, $devicefull);
4208         eval { qemu_deviceaddverify($vmid, $deviceid); };
4209         if (my $err = $@) {
4210             eval { qemu_drivedel($vmid, $deviceid); };
4211             warn $@ if $@;
4212             die $err;
4213         }
4214     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4215         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4216         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4217         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4218
4219         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4220
4221         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4222             qemu_iothread_add($vmid, $deviceid, $device);
4223             $devicefull .= ",iothread=iothread-$deviceid";
4224         }
4225
4226         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4227             $devicefull .= ",num_queues=$device->{queues}";
4228         }
4229
4230         qemu_deviceadd($vmid, $devicefull);
4231         qemu_deviceaddverify($vmid, $deviceid);
4232     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4233         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4234         qemu_driveadd($storecfg, $vmid, $device);
4235
4236         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4237         eval { qemu_deviceadd($vmid, $devicefull); };
4238         if (my $err = $@) {
4239             eval { qemu_drivedel($vmid, $deviceid); };
4240             warn $@ if $@;
4241             die $err;
4242         }
4243     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4244         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4245
4246         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4247         my $use_old_bios_files = undef;
4248         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4249
4250         my $netdevicefull = print_netdevice_full(
4251             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4252         qemu_deviceadd($vmid, $netdevicefull);
4253         eval {
4254             qemu_deviceaddverify($vmid, $deviceid);
4255             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4256         };
4257         if (my $err = $@) {
4258             eval { qemu_netdevdel($vmid, $deviceid); };
4259             warn $@ if $@;
4260             die $err;
4261         }
4262     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4263         my $bridgeid = $2;
4264         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4265         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4266
4267         qemu_deviceadd($vmid, $devicefull);
4268         qemu_deviceaddverify($vmid, $deviceid);
4269     } else {
4270         die "can't hotplug device '$deviceid'\n";
4271     }
4272
4273     return 1;
4274 }
4275
4276 # fixme: this should raise exceptions on error!
4277 sub vm_deviceunplug {
4278     my ($vmid, $conf, $deviceid) = @_;
4279
4280     my $devices_list = vm_devices_list($vmid);
4281     return 1 if !defined($devices_list->{$deviceid});
4282
4283     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4284     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4285
4286     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4287         qemu_devicedel($vmid, $deviceid);
4288     } elsif ($deviceid =~ m/^usb\d+$/) {
4289         die "usb hotplug currently not reliable\n";
4290         # when unplugging usb devices this way, there may be remaining usb
4291         # controllers/hubs so we disable it for now
4292         #qemu_devicedel($vmid, $deviceid);
4293         #qemu_devicedelverify($vmid, $deviceid);
4294     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4295         my $device = parse_drive($deviceid, $conf->{$deviceid});
4296
4297         qemu_devicedel($vmid, $deviceid);
4298         qemu_devicedelverify($vmid, $deviceid);
4299         qemu_drivedel($vmid, $deviceid);
4300         qemu_iothread_del($vmid, $deviceid, $device);
4301     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4302         qemu_devicedel($vmid, $deviceid);
4303         qemu_devicedelverify($vmid, $deviceid);
4304     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4305         my $device = parse_drive($deviceid, $conf->{$deviceid});
4306
4307         qemu_devicedel($vmid, $deviceid);
4308         qemu_devicedelverify($vmid, $deviceid);
4309         qemu_drivedel($vmid, $deviceid);
4310         qemu_deletescsihw($conf, $vmid, $deviceid);
4311
4312         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4313             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4314     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4315         qemu_devicedel($vmid, $deviceid);
4316         qemu_devicedelverify($vmid, $deviceid);
4317         qemu_netdevdel($vmid, $deviceid);
4318     } else {
4319         die "can't unplug device '$deviceid'\n";
4320     }
4321
4322     return 1;
4323 }
4324
4325 sub qemu_deviceadd {
4326     my ($vmid, $devicefull) = @_;
4327
4328     $devicefull = "driver=".$devicefull;
4329     my %options =  split(/[=,]/, $devicefull);
4330
4331     mon_cmd($vmid, "device_add" , %options);
4332 }
4333
4334 sub qemu_devicedel {
4335     my ($vmid, $deviceid) = @_;
4336
4337     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4338 }
4339
4340 sub qemu_iothread_add {
4341     my ($vmid, $deviceid, $device) = @_;
4342
4343     if ($device->{iothread}) {
4344         my $iothreads = vm_iothreads_list($vmid);
4345         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4346     }
4347 }
4348
4349 sub qemu_iothread_del {
4350     my ($vmid, $deviceid, $device) = @_;
4351
4352     if ($device->{iothread}) {
4353         my $iothreads = vm_iothreads_list($vmid);
4354         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4355     }
4356 }
4357
4358 sub qemu_objectadd {
4359     my ($vmid, $objectid, $qomtype) = @_;
4360
4361     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4362
4363     return 1;
4364 }
4365
4366 sub qemu_objectdel {
4367     my ($vmid, $objectid) = @_;
4368
4369     mon_cmd($vmid, "object-del", id => $objectid);
4370
4371     return 1;
4372 }
4373
4374 sub qemu_driveadd {
4375     my ($storecfg, $vmid, $device) = @_;
4376
4377     my $kvmver = get_running_qemu_version($vmid);
4378     my $io_uring = min_version($kvmver, 6, 0);
4379     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4380     $drive =~ s/\\/\\\\/g;
4381     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4382
4383     # If the command succeeds qemu prints: "OK"
4384     return 1 if $ret =~ m/OK/s;
4385
4386     die "adding drive failed: $ret\n";
4387 }
4388
4389 sub qemu_drivedel {
4390     my ($vmid, $deviceid) = @_;
4391
4392     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4393     $ret =~ s/^\s+//;
4394
4395     return 1 if $ret eq "";
4396
4397     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4398     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4399
4400     die "deleting drive $deviceid failed : $ret\n";
4401 }
4402
4403 sub qemu_deviceaddverify {
4404     my ($vmid, $deviceid) = @_;
4405
4406     for (my $i = 0; $i <= 5; $i++) {
4407          my $devices_list = vm_devices_list($vmid);
4408          return 1 if defined($devices_list->{$deviceid});
4409          sleep 1;
4410     }
4411
4412     die "error on hotplug device '$deviceid'\n";
4413 }
4414
4415
4416 sub qemu_devicedelverify {
4417     my ($vmid, $deviceid) = @_;
4418
4419     # need to verify that the device is correctly removed as device_del
4420     # is async and empty return is not reliable
4421
4422     for (my $i = 0; $i <= 5; $i++) {
4423          my $devices_list = vm_devices_list($vmid);
4424          return 1 if !defined($devices_list->{$deviceid});
4425          sleep 1;
4426     }
4427
4428     die "error on hot-unplugging device '$deviceid'\n";
4429 }
4430
4431 sub qemu_findorcreatescsihw {
4432     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4433
4434     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4435
4436     my $scsihwid="$controller_prefix$controller";
4437     my $devices_list = vm_devices_list($vmid);
4438
4439     if (!defined($devices_list->{$scsihwid})) {
4440         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4441     }
4442
4443     return 1;
4444 }
4445
4446 sub qemu_deletescsihw {
4447     my ($conf, $vmid, $opt) = @_;
4448
4449     my $device = parse_drive($opt, $conf->{$opt});
4450
4451     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4452         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4453         return 1;
4454     }
4455
4456     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4457
4458     my $devices_list = vm_devices_list($vmid);
4459     foreach my $opt (keys %{$devices_list}) {
4460         if (is_valid_drivename($opt)) {
4461             my $drive = parse_drive($opt, $conf->{$opt});
4462             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4463                 return 1;
4464             }
4465         }
4466     }
4467
4468     my $scsihwid="scsihw$controller";
4469
4470     vm_deviceunplug($vmid, $conf, $scsihwid);
4471
4472     return 1;
4473 }
4474
4475 sub qemu_add_pci_bridge {
4476     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4477
4478     my $bridges = {};
4479
4480     my $bridgeid;
4481
4482     print_pci_addr($device, $bridges, $arch, $machine_type);
4483
4484     while (my ($k, $v) = each %$bridges) {
4485         $bridgeid = $k;
4486     }
4487     return 1 if !defined($bridgeid) || $bridgeid < 1;
4488
4489     my $bridge = "pci.$bridgeid";
4490     my $devices_list = vm_devices_list($vmid);
4491
4492     if (!defined($devices_list->{$bridge})) {
4493         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4494     }
4495
4496     return 1;
4497 }
4498
4499 sub qemu_set_link_status {
4500     my ($vmid, $device, $up) = @_;
4501
4502     mon_cmd($vmid, "set_link", name => $device,
4503                up => $up ? JSON::true : JSON::false);
4504 }
4505
4506 sub qemu_netdevadd {
4507     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4508
4509     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4510     my %options =  split(/[=,]/, $netdev);
4511
4512     if (defined(my $vhost = $options{vhost})) {
4513         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4514     }
4515
4516     if (defined(my $queues = $options{queues})) {
4517         $options{queues} = $queues + 0;
4518     }
4519
4520     mon_cmd($vmid, "netdev_add",  %options);
4521     return 1;
4522 }
4523
4524 sub qemu_netdevdel {
4525     my ($vmid, $deviceid) = @_;
4526
4527     mon_cmd($vmid, "netdev_del", id => $deviceid);
4528 }
4529
4530 sub qemu_usb_hotplug {
4531     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4532
4533     return if !$device;
4534
4535     # remove the old one first
4536     vm_deviceunplug($vmid, $conf, $deviceid);
4537
4538     # check if xhci controller is necessary and available
4539     if ($device->{usb3}) {
4540
4541         my $devicelist = vm_devices_list($vmid);
4542
4543         if (!$devicelist->{xhci}) {
4544             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4545             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4546         }
4547     }
4548     my $d = parse_usb_device($device->{host});
4549     $d->{usb3} = $device->{usb3};
4550
4551     # add the new one
4552     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4553 }
4554
4555 sub qemu_cpu_hotplug {
4556     my ($vmid, $conf, $vcpus) = @_;
4557
4558     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4559
4560     my $sockets = 1;
4561     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4562     $sockets = $conf->{sockets} if  $conf->{sockets};
4563     my $cores = $conf->{cores} || 1;
4564     my $maxcpus = $sockets * $cores;
4565
4566     $vcpus = $maxcpus if !$vcpus;
4567
4568     die "you can't add more vcpus than maxcpus\n"
4569         if $vcpus > $maxcpus;
4570
4571     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4572
4573     if ($vcpus < $currentvcpus) {
4574
4575         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4576
4577             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4578                 qemu_devicedel($vmid, "cpu$i");
4579                 my $retry = 0;
4580                 my $currentrunningvcpus = undef;
4581                 while (1) {
4582                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4583                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4584                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4585                     $retry++;
4586                     sleep 1;
4587                 }
4588                 #update conf after each succesfull cpu unplug
4589                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4590                 PVE::QemuConfig->write_config($vmid, $conf);
4591             }
4592         } else {
4593             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4594         }
4595
4596         return;
4597     }
4598
4599     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600     die "vcpus in running vm does not match its configuration\n"
4601         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4602
4603     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4604
4605         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4606             my $cpustr = print_cpu_device($conf, $i);
4607             qemu_deviceadd($vmid, $cpustr);
4608
4609             my $retry = 0;
4610             my $currentrunningvcpus = undef;
4611             while (1) {
4612                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4613                 last if scalar(@{$currentrunningvcpus}) == $i;
4614                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4615                 sleep 1;
4616                 $retry++;
4617             }
4618             #update conf after each succesfull cpu hotplug
4619             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4620             PVE::QemuConfig->write_config($vmid, $conf);
4621         }
4622     } else {
4623
4624         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4625             mon_cmd($vmid, "cpu-add", id => int($i));
4626         }
4627     }
4628 }
4629
4630 sub qemu_block_set_io_throttle {
4631     my ($vmid, $deviceid,
4632         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4633         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4634         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4635         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4636
4637     return if !check_running($vmid) ;
4638
4639     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4640         bps => int($bps),
4641         bps_rd => int($bps_rd),
4642         bps_wr => int($bps_wr),
4643         iops => int($iops),
4644         iops_rd => int($iops_rd),
4645         iops_wr => int($iops_wr),
4646         bps_max => int($bps_max),
4647         bps_rd_max => int($bps_rd_max),
4648         bps_wr_max => int($bps_wr_max),
4649         iops_max => int($iops_max),
4650         iops_rd_max => int($iops_rd_max),
4651         iops_wr_max => int($iops_wr_max),
4652         bps_max_length => int($bps_max_length),
4653         bps_rd_max_length => int($bps_rd_max_length),
4654         bps_wr_max_length => int($bps_wr_max_length),
4655         iops_max_length => int($iops_max_length),
4656         iops_rd_max_length => int($iops_rd_max_length),
4657         iops_wr_max_length => int($iops_wr_max_length),
4658     );
4659
4660 }
4661
4662 sub qemu_block_resize {
4663     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4664
4665     my $running = check_running($vmid);
4666
4667     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4668
4669     return if !$running;
4670
4671     my $padding = (1024 - $size % 1024) % 1024;
4672     $size = $size + $padding;
4673
4674     mon_cmd(
4675         $vmid,
4676         "block_resize",
4677         device => $deviceid,
4678         size => int($size),
4679         timeout => 60,
4680     );
4681 }
4682
4683 sub qemu_volume_snapshot {
4684     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4685
4686     my $running = check_running($vmid);
4687
4688     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4689         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4690     } else {
4691         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4692     }
4693 }
4694
4695 sub qemu_volume_snapshot_delete {
4696     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4697
4698     my $running = check_running($vmid);
4699
4700     if($running) {
4701
4702         $running = undef;
4703         my $conf = PVE::QemuConfig->load_config($vmid);
4704         PVE::QemuConfig->foreach_volume($conf, sub {
4705             my ($ds, $drive) = @_;
4706             $running = 1 if $drive->{file} eq $volid;
4707         });
4708     }
4709
4710     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4711         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4712     } else {
4713         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4714     }
4715 }
4716
4717 sub set_migration_caps {
4718     my ($vmid, $savevm) = @_;
4719
4720     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4721
4722     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4723     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4724
4725     my $cap_ref = [];
4726
4727     my $enabled_cap = {
4728         "auto-converge" => 1,
4729         "xbzrle" => 1,
4730         "x-rdma-pin-all" => 0,
4731         "zero-blocks" => 0,
4732         "compress" => 0,
4733         "dirty-bitmaps" => $dirty_bitmaps,
4734     };
4735
4736     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4737
4738     for my $supported_capability (@$supported_capabilities) {
4739         push @$cap_ref, {
4740             capability => $supported_capability->{capability},
4741             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4742         };
4743     }
4744
4745     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4746 }
4747
4748 sub foreach_volid {
4749     my ($conf, $func, @param) = @_;
4750
4751     my $volhash = {};
4752
4753     my $test_volid = sub {
4754         my ($key, $drive, $snapname) = @_;
4755
4756         my $volid = $drive->{file};
4757         return if !$volid;
4758
4759         $volhash->{$volid}->{cdrom} //= 1;
4760         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4761
4762         my $replicate = $drive->{replicate} // 1;
4763         $volhash->{$volid}->{replicate} //= 0;
4764         $volhash->{$volid}->{replicate} = 1 if $replicate;
4765
4766         $volhash->{$volid}->{shared} //= 0;
4767         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4768
4769         $volhash->{$volid}->{referenced_in_config} //= 0;
4770         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4771
4772         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4773             if defined($snapname);
4774
4775         my $size = $drive->{size};
4776         $volhash->{$volid}->{size} //= $size if $size;
4777
4778         $volhash->{$volid}->{is_vmstate} //= 0;
4779         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4780
4781         $volhash->{$volid}->{is_tpmstate} //= 0;
4782         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4783
4784         $volhash->{$volid}->{is_unused} //= 0;
4785         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4786
4787         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4788     };
4789
4790     my $include_opts = {
4791         extra_keys => ['vmstate'],
4792         include_unused => 1,
4793     };
4794
4795     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4796     foreach my $snapname (keys %{$conf->{snapshots}}) {
4797         my $snap = $conf->{snapshots}->{$snapname};
4798         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4799     }
4800
4801     foreach my $volid (keys %$volhash) {
4802         &$func($volid, $volhash->{$volid}, @param);
4803     }
4804 }
4805
4806 my $fast_plug_option = {
4807     'lock' => 1,
4808     'name' => 1,
4809     'onboot' => 1,
4810     'shares' => 1,
4811     'startup' => 1,
4812     'description' => 1,
4813     'protection' => 1,
4814     'vmstatestorage' => 1,
4815     'hookscript' => 1,
4816     'tags' => 1,
4817 };
4818
4819 for my $opt (keys %$confdesc_cloudinit) {
4820     $fast_plug_option->{$opt} = 1;
4821 };
4822
4823 # hotplug changes in [PENDING]
4824 # $selection hash can be used to only apply specified options, for
4825 # example: { cores => 1 } (only apply changed 'cores')
4826 # $errors ref is used to return error messages
4827 sub vmconfig_hotplug_pending {
4828     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4829
4830     my $defaults = load_defaults();
4831     my $arch = get_vm_arch($conf);
4832     my $machine_type = get_vm_machine($conf, undef, $arch);
4833
4834     # commit values which do not have any impact on running VM first
4835     # Note: those option cannot raise errors, we we do not care about
4836     # $selection and always apply them.
4837
4838     my $add_error = sub {
4839         my ($opt, $msg) = @_;
4840         $errors->{$opt} = "hotplug problem - $msg";
4841     };
4842
4843     my $changes = 0;
4844     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4845         if ($fast_plug_option->{$opt}) {
4846             $conf->{$opt} = $conf->{pending}->{$opt};
4847             delete $conf->{pending}->{$opt};
4848             $changes = 1;
4849         }
4850     }
4851
4852     if ($changes) {
4853         PVE::QemuConfig->write_config($vmid, $conf);
4854     }
4855
4856     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4857
4858     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4859     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4860     foreach my $opt (sort keys %$pending_delete_hash) {
4861         next if $selection && !$selection->{$opt};
4862         my $force = $pending_delete_hash->{$opt}->{force};
4863         eval {
4864             if ($opt eq 'hotplug') {
4865                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4866             } elsif ($opt eq 'tablet') {
4867                 die "skip\n" if !$hotplug_features->{usb};
4868                 if ($defaults->{tablet}) {
4869                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4870                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4871                         if $arch eq 'aarch64';
4872                 } else {
4873                     vm_deviceunplug($vmid, $conf, 'tablet');
4874                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4875                 }
4876             } elsif ($opt =~ m/^usb\d+/) {
4877                 die "skip\n";
4878                 # since we cannot reliably hot unplug usb devices we are disabling it
4879                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4880                 #vm_deviceunplug($vmid, $conf, $opt);
4881             } elsif ($opt eq 'vcpus') {
4882                 die "skip\n" if !$hotplug_features->{cpu};
4883                 qemu_cpu_hotplug($vmid, $conf, undef);
4884             } elsif ($opt eq 'balloon') {
4885                 # enable balloon device is not hotpluggable
4886                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4887                 # here we reset the ballooning value to memory
4888                 my $balloon = $conf->{memory} || $defaults->{memory};
4889                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4890             } elsif ($fast_plug_option->{$opt}) {
4891                 # do nothing
4892             } elsif ($opt =~ m/^net(\d+)$/) {
4893                 die "skip\n" if !$hotplug_features->{network};
4894                 vm_deviceunplug($vmid, $conf, $opt);
4895             } elsif (is_valid_drivename($opt)) {
4896                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4897                 vm_deviceunplug($vmid, $conf, $opt);
4898                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4899             } elsif ($opt =~ m/^memory$/) {
4900                 die "skip\n" if !$hotplug_features->{memory};
4901                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4902             } elsif ($opt eq 'cpuunits') {
4903                 $cgroup->change_cpu_shares(undef);
4904             } elsif ($opt eq 'cpulimit') {
4905                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4906             } else {
4907                 die "skip\n";
4908             }
4909         };
4910         if (my $err = $@) {
4911             &$add_error($opt, $err) if $err ne "skip\n";
4912         } else {
4913             delete $conf->{$opt};
4914             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4915         }
4916     }
4917
4918     foreach my $opt (keys %{$conf->{pending}}) {
4919         next if $selection && !$selection->{$opt};
4920         my $value = $conf->{pending}->{$opt};
4921         eval {
4922             if ($opt eq 'hotplug') {
4923                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4924             } elsif ($opt eq 'tablet') {
4925                 die "skip\n" if !$hotplug_features->{usb};
4926                 if ($value == 1) {
4927                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4928                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4929                         if $arch eq 'aarch64';
4930                 } elsif ($value == 0) {
4931                     vm_deviceunplug($vmid, $conf, 'tablet');
4932                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4933                 }
4934             } elsif ($opt =~ m/^usb\d+$/) {
4935                 die "skip\n";
4936                 # since we cannot reliably hot unplug usb devices we disable it for now
4937                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4938                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4939                 #die "skip\n" if !$d;
4940                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4941             } elsif ($opt eq 'vcpus') {
4942                 die "skip\n" if !$hotplug_features->{cpu};
4943                 qemu_cpu_hotplug($vmid, $conf, $value);
4944             } elsif ($opt eq 'balloon') {
4945                 # enable/disable balloning device is not hotpluggable
4946                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4947                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4948                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4949
4950                 # allow manual ballooning if shares is set to zero
4951                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4952                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4953                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4954                 }
4955             } elsif ($opt =~ m/^net(\d+)$/) {
4956                 # some changes can be done without hotplug
4957                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4958                                     $vmid, $opt, $value, $arch, $machine_type);
4959             } elsif (is_valid_drivename($opt)) {
4960                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4961                 # some changes can be done without hotplug
4962                 my $drive = parse_drive($opt, $value);
4963                 if (drive_is_cloudinit($drive)) {
4964                     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4965                 }
4966                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4967                                      $vmid, $opt, $value, $arch, $machine_type);
4968             } elsif ($opt =~ m/^memory$/) { #dimms
4969                 die "skip\n" if !$hotplug_features->{memory};
4970                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4971             } elsif ($opt eq 'cpuunits') {
4972                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
4973                 $cgroup->change_cpu_shares($new_cpuunits);
4974             } elsif ($opt eq 'cpulimit') {
4975                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4976                 $cgroup->change_cpu_quota($cpulimit, 100000);
4977             } elsif ($opt eq 'agent') {
4978                 vmconfig_update_agent($conf, $opt, $value);
4979             } else {
4980                 die "skip\n";  # skip non-hot-pluggable options
4981             }
4982         };
4983         if (my $err = $@) {
4984             &$add_error($opt, $err) if $err ne "skip\n";
4985         } else {
4986             $conf->{$opt} = $value;
4987             delete $conf->{pending}->{$opt};
4988         }
4989     }
4990
4991     PVE::QemuConfig->write_config($vmid, $conf);
4992 }
4993
4994 sub try_deallocate_drive {
4995     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4996
4997     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4998         my $volid = $drive->{file};
4999         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5000             my $sid = PVE::Storage::parse_volume_id($volid);
5001             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5002
5003             # check if the disk is really unused
5004             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5005                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5006             PVE::Storage::vdisk_free($storecfg, $volid);
5007             return 1;
5008         } else {
5009             # If vm is not owner of this disk remove from config
5010             return 1;
5011         }
5012     }
5013
5014     return;
5015 }
5016
5017 sub vmconfig_delete_or_detach_drive {
5018     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5019
5020     my $drive = parse_drive($opt, $conf->{$opt});
5021
5022     my $rpcenv = PVE::RPCEnvironment::get();
5023     my $authuser = $rpcenv->get_user();
5024
5025     if ($force) {
5026         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5027         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5028     } else {
5029         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5030     }
5031 }
5032
5033
5034
5035 sub vmconfig_apply_pending {
5036     my ($vmid, $conf, $storecfg, $errors) = @_;
5037
5038     return if !scalar(keys %{$conf->{pending}});
5039
5040     my $add_apply_error = sub {
5041         my ($opt, $msg) = @_;
5042         my $err_msg = "unable to apply pending change $opt : $msg";
5043         $errors->{$opt} = $err_msg;
5044         warn $err_msg;
5045     };
5046
5047     # cold plug
5048
5049     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5050     foreach my $opt (sort keys %$pending_delete_hash) {
5051         my $force = $pending_delete_hash->{$opt}->{force};
5052         eval {
5053             if ($opt =~ m/^unused/) {
5054                 die "internal error";
5055             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5056                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5057             }
5058         };
5059         if (my $err = $@) {
5060             $add_apply_error->($opt, $err);
5061         } else {
5062             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5063             delete $conf->{$opt};
5064         }
5065     }
5066
5067     PVE::QemuConfig->cleanup_pending($conf);
5068
5069     my $generate_cloudnit = undef;
5070
5071     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5072         next if $opt eq 'delete'; # just to be sure
5073         eval {
5074             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5075                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5076             }
5077         };
5078         if (my $err = $@) {
5079             $add_apply_error->($opt, $err);
5080         } else {
5081
5082             if (is_valid_drivename($opt)) {
5083                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5084                 $generate_cloudnit = 1 if drive_is_cloudinit($drive);
5085             }
5086
5087             $conf->{$opt} = delete $conf->{pending}->{$opt};
5088         }
5089     }
5090
5091     # write all changes at once to avoid unnecessary i/o
5092     PVE::QemuConfig->write_config($vmid, $conf);
5093     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if $generate_cloudnit;
5094 }
5095
5096 sub vmconfig_update_net {
5097     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5098
5099     my $newnet = parse_net($value);
5100
5101     if ($conf->{$opt}) {
5102         my $oldnet = parse_net($conf->{$opt});
5103
5104         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5105             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5106             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5107             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5108
5109             # for non online change, we try to hot-unplug
5110             die "skip\n" if !$hotplug;
5111             vm_deviceunplug($vmid, $conf, $opt);
5112         } else {
5113
5114             die "internal error" if $opt !~ m/net(\d+)/;
5115             my $iface = "tap${vmid}i$1";
5116
5117             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5118                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5119                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5120                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5121                 PVE::Network::tap_unplug($iface);
5122
5123                 if ($have_sdn) {
5124                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5125                 } else {
5126                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5127                 }
5128             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5129                 # Rate can be applied on its own but any change above needs to
5130                 # include the rate in tap_plug since OVS resets everything.
5131                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5132             }
5133
5134             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5135                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5136             }
5137
5138             return 1;
5139         }
5140     }
5141
5142     if ($hotplug) {
5143         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5144     } else {
5145         die "skip\n";
5146     }
5147 }
5148
5149 sub vmconfig_update_agent {
5150     my ($conf, $opt, $value) = @_;
5151
5152     die "skip\n" if !$conf->{$opt};
5153
5154     my $hotplug_options = { fstrim_cloned_disks => 1 };
5155
5156     my $old_agent = parse_guest_agent($conf);
5157     my $agent = parse_guest_agent({$opt => $value});
5158
5159     for my $option (keys %$agent) { # added/changed options
5160         next if defined($hotplug_options->{$option});
5161         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5162     }
5163
5164     for my $option (keys %$old_agent) { # removed options
5165         next if defined($hotplug_options->{$option});
5166         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5167     }
5168
5169     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5170 }
5171
5172 sub vmconfig_update_disk {
5173     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5174
5175     my $drive = parse_drive($opt, $value);
5176
5177     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5178         my $media = $drive->{media} || 'disk';
5179         my $oldmedia = $old_drive->{media} || 'disk';
5180         die "unable to change media type\n" if $media ne $oldmedia;
5181
5182         if (!drive_is_cdrom($old_drive)) {
5183
5184             if ($drive->{file} ne $old_drive->{file}) {
5185
5186                 die "skip\n" if !$hotplug;
5187
5188                 # unplug and register as unused
5189                 vm_deviceunplug($vmid, $conf, $opt);
5190                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5191
5192             } else {
5193                 # update existing disk
5194
5195                 # skip non hotpluggable value
5196                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5197                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5198                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5199                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5200                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5201                     die "skip\n";
5202                 }
5203
5204                 # apply throttle
5205                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5206                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5207                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5208                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5209                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5210                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5211                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5212                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5213                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5214                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5215                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5216                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5217                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5218                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5219                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5220                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5221                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5222                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5223
5224                     qemu_block_set_io_throttle(
5225                         $vmid,"drive-$opt",
5226                         ($drive->{mbps} || 0)*1024*1024,
5227                         ($drive->{mbps_rd} || 0)*1024*1024,
5228                         ($drive->{mbps_wr} || 0)*1024*1024,
5229                          $drive->{iops} || 0,
5230                          $drive->{iops_rd} || 0,
5231                          $drive->{iops_wr} || 0,
5232                         ($drive->{mbps_max} || 0)*1024*1024,
5233                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5234                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5235                          $drive->{iops_max} || 0,
5236                          $drive->{iops_rd_max} || 0,
5237                          $drive->{iops_wr_max} || 0,
5238                          $drive->{bps_max_length} || 1,
5239                          $drive->{bps_rd_max_length} || 1,
5240                          $drive->{bps_wr_max_length} || 1,
5241                          $drive->{iops_max_length} || 1,
5242                          $drive->{iops_rd_max_length} || 1,
5243                          $drive->{iops_wr_max_length} || 1,
5244                     );
5245
5246                 }
5247
5248                 return 1;
5249             }
5250
5251         } else { # cdrom
5252
5253             if ($drive->{file} eq 'none') {
5254                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5255                 if (drive_is_cloudinit($old_drive)) {
5256                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5257                 }
5258             } else {
5259                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5260
5261                 # force eject if locked
5262                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5263
5264                 if ($path) {
5265                     mon_cmd($vmid, "blockdev-change-medium",
5266                         id => "$opt", filename => "$path");
5267                 }
5268             }
5269
5270             return 1;
5271         }
5272     }
5273
5274     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5275     # hotplug new disks
5276     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5277     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5278 }
5279
5280 # called in locked context by incoming migration
5281 sub vm_migrate_get_nbd_disks {
5282     my ($storecfg, $conf, $replicated_volumes) = @_;
5283
5284     my $local_volumes = {};
5285     PVE::QemuConfig->foreach_volume($conf, sub {
5286         my ($ds, $drive) = @_;
5287
5288         return if drive_is_cdrom($drive);
5289         return if $ds eq 'tpmstate0';
5290
5291         my $volid = $drive->{file};
5292
5293         return if !$volid;
5294
5295         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5296
5297         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5298         return if $scfg->{shared};
5299
5300         # replicated disks re-use existing state via bitmap
5301         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5302         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5303     });
5304     return $local_volumes;
5305 }
5306
5307 # called in locked context by incoming migration
5308 sub vm_migrate_alloc_nbd_disks {
5309     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5310
5311     my $nbd = {};
5312     foreach my $opt (sort keys %$source_volumes) {
5313         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5314
5315         if ($use_existing) {
5316             $nbd->{$opt}->{drivestr} = print_drive($drive);
5317             $nbd->{$opt}->{volid} = $volid;
5318             $nbd->{$opt}->{replicated} = 1;
5319             next;
5320         }
5321
5322         # storage mapping + volname = regular migration
5323         # storage mapping + format = remote migration
5324         # order of precedence, filtered by whether storage supports it:
5325         # 1. explicit requested format
5326         # 2. format of current volume
5327         # 3. default format of storage
5328         if (!$storagemap->{identity}) {
5329             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5330             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5331             if (!$format || !grep { $format eq $_ } @$validFormats) {
5332                 if ($volname) {
5333                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5334                     my $fileFormat = qemu_img_format($scfg, $volname);
5335                     $format = $fileFormat
5336                         if grep { $fileFormat eq $_ } @$validFormats;
5337                 }
5338                 $format //= $defFormat;
5339             }
5340         } else {
5341             # can't happen for remote migration, so $volname is always defined
5342             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5343             $format = qemu_img_format($scfg, $volname);
5344         }
5345
5346         my $size = $drive->{size} / 1024;
5347         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5348         my $newdrive = $drive;
5349         $newdrive->{format} = $format;
5350         $newdrive->{file} = $newvolid;
5351         my $drivestr = print_drive($newdrive);
5352         $nbd->{$opt}->{drivestr} = $drivestr;
5353         $nbd->{$opt}->{volid} = $newvolid;
5354     }
5355
5356     return $nbd;
5357 }
5358
5359 # see vm_start_nolock for parameters, additionally:
5360 # migrate_opts:
5361 #   storagemap = parsed storage map for allocating NBD disks
5362 sub vm_start {
5363     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5364
5365     return PVE::QemuConfig->lock_config($vmid, sub {
5366         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5367
5368         die "you can't start a vm if it's a template\n"
5369             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5370
5371         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5372         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5373
5374         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5375
5376         if ($has_backup_lock && $running) {
5377             # a backup is currently running, attempt to start the guest in the
5378             # existing QEMU instance
5379             return vm_resume($vmid);
5380         }
5381
5382         PVE::QemuConfig->check_lock($conf)
5383             if !($params->{skiplock} || $has_suspended_lock);
5384
5385         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5386
5387         die "VM $vmid already running\n" if $running;
5388
5389         if (my $storagemap = $migrate_opts->{storagemap}) {
5390             my $replicated = $migrate_opts->{replicated_volumes};
5391             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5392             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5393
5394             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5395                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5396             }
5397         }
5398
5399         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5400     });
5401 }
5402
5403
5404 # params:
5405 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5406 #   skiplock => 0/1, skip checking for config lock
5407 #   skiptemplate => 0/1, skip checking whether VM is template
5408 #   forcemachine => to force Qemu machine (rollback/migration)
5409 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5410 #   timeout => in seconds
5411 #   paused => start VM in paused state (backup)
5412 #   resume => resume from hibernation
5413 #   pbs-backing => {
5414 #      sata0 => {
5415 #         repository
5416 #         snapshot
5417 #         keyfile
5418 #         archive
5419 #      },
5420 #      virtio2 => ...
5421 #   }
5422 # migrate_opts:
5423 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5424 #   migratedfrom => source node
5425 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5426 #   network => CIDR of migration network
5427 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5428 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5429 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5430 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5431 #       contained in config
5432 sub vm_start_nolock {
5433     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5434
5435     my $statefile = $params->{statefile};
5436     my $resume = $params->{resume};
5437
5438     my $migratedfrom = $migrate_opts->{migratedfrom};
5439     my $migration_type = $migrate_opts->{type};
5440
5441     my $res = {};
5442
5443     # clean up leftover reboot request files
5444     eval { clear_reboot_request($vmid); };
5445     warn $@ if $@;
5446
5447     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5448         vmconfig_apply_pending($vmid, $conf, $storecfg);
5449         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5450     }
5451
5452     # don't regenerate the ISO if the VM is started as part of a live migration
5453     # this way we can reuse the old ISO with the correct config
5454     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5455
5456     # override offline migrated volumes, conf is out of date still
5457     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5458         for my $key (sort keys $offline_volumes->%*) {
5459             my $parsed = parse_drive($key, $conf->{$key});
5460             $parsed->{file} = $offline_volumes->{$key};
5461             $conf->{$key} = print_drive($parsed);
5462         }
5463     }
5464
5465     my $defaults = load_defaults();
5466
5467     # set environment variable useful inside network script
5468     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5469
5470     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5471
5472     my $forcemachine = $params->{forcemachine};
5473     my $forcecpu = $params->{forcecpu};
5474     if ($resume) {
5475         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5476         $forcemachine = $conf->{runningmachine};
5477         $forcecpu = $conf->{runningcpu};
5478         print "Resuming suspended VM\n";
5479     }
5480
5481     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5482         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5483
5484     my $migration_ip;
5485     my $get_migration_ip = sub {
5486         my ($nodename) = @_;
5487
5488         return $migration_ip if defined($migration_ip);
5489
5490         my $cidr = $migrate_opts->{network};
5491
5492         if (!defined($cidr)) {
5493             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5494             $cidr = $dc_conf->{migration}->{network};
5495         }
5496
5497         if (defined($cidr)) {
5498             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5499
5500             die "could not get IP: no address configured on local " .
5501                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5502
5503             die "could not get IP: multiple addresses configured on local " .
5504                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5505
5506             $migration_ip = @$ips[0];
5507         }
5508
5509         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5510             if !defined($migration_ip);
5511
5512         return $migration_ip;
5513     };
5514
5515     my $migrate_uri;
5516     if ($statefile) {
5517         if ($statefile eq 'tcp') {
5518             my $localip = "localhost";
5519             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5520             my $nodename = nodename();
5521
5522             if (!defined($migration_type)) {
5523                 if (defined($datacenterconf->{migration}->{type})) {
5524                     $migration_type = $datacenterconf->{migration}->{type};
5525                 } else {
5526                     $migration_type = 'secure';
5527                 }
5528             }
5529
5530             if ($migration_type eq 'insecure') {
5531                 $localip = $get_migration_ip->($nodename);
5532                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5533             }
5534
5535             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5536             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5537             $migrate_uri = "tcp:${localip}:${migrate_port}";
5538             push @$cmd, '-incoming', $migrate_uri;
5539             push @$cmd, '-S';
5540
5541         } elsif ($statefile eq 'unix') {
5542             # should be default for secure migrations as a ssh TCP forward
5543             # tunnel is not deterministic reliable ready and fails regurarly
5544             # to set up in time, so use UNIX socket forwards
5545             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5546             unlink $socket_addr;
5547
5548             $migrate_uri = "unix:$socket_addr";
5549
5550             push @$cmd, '-incoming', $migrate_uri;
5551             push @$cmd, '-S';
5552
5553         } elsif (-e $statefile) {
5554             push @$cmd, '-loadstate', $statefile;
5555         } else {
5556             my $statepath = PVE::Storage::path($storecfg, $statefile);
5557             push @$vollist, $statefile;
5558             push @$cmd, '-loadstate', $statepath;
5559         }
5560     } elsif ($params->{paused}) {
5561         push @$cmd, '-S';
5562     }
5563
5564     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5565
5566     my $pci_devices = {}; # host pci devices
5567     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5568         my $dev = $conf->{"hostpci$i"} or next;
5569         $pci_devices->{$i} = parse_hostpci($dev);
5570     }
5571
5572     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5573     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5574
5575     # map to a flat list of pci ids
5576     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5577
5578     # reserve all PCI IDs before actually doing anything with them
5579     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5580
5581     eval {
5582         my $uuid;
5583         for my $id (sort keys %$pci_devices) {
5584             my $d = $pci_devices->{$id};
5585             for my $dev ($d->{pciid}->@*) {
5586                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5587
5588                 # nvidia grid needs the uuid of the mdev as qemu parameter
5589                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5590                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5591                 }
5592             }
5593         }
5594         push @$cmd, '-uuid', $uuid if defined($uuid);
5595     };
5596     if (my $err = $@) {
5597         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5598         warn $@ if $@;
5599         die $err;
5600     }
5601
5602     PVE::Storage::activate_volumes($storecfg, $vollist);
5603
5604     eval {
5605         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5606     };
5607     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5608     # timeout should be more than enough here...
5609     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5610
5611     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5612
5613     my %run_params = (
5614         timeout => $statefile ? undef : $start_timeout,
5615         umask => 0077,
5616         noerr => 1,
5617     );
5618
5619     # when migrating, prefix QEMU output so other side can pick up any
5620     # errors that might occur and show the user
5621     if ($migratedfrom) {
5622         $run_params{quiet} = 1;
5623         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5624     }
5625
5626     my %systemd_properties = (
5627         Slice => 'qemu.slice',
5628         KillMode => 'process',
5629         SendSIGKILL => 0,
5630         TimeoutStopUSec => ULONG_MAX, # infinity
5631     );
5632
5633     if (PVE::CGroup::cgroup_mode() == 2) {
5634         $systemd_properties{CPUWeight} = $cpuunits;
5635     } else {
5636         $systemd_properties{CPUShares} = $cpuunits;
5637     }
5638
5639     if (my $cpulimit = $conf->{cpulimit}) {
5640         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5641     }
5642     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5643
5644     my $run_qemu = sub {
5645         PVE::Tools::run_fork sub {
5646             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5647
5648             my $tpmpid;
5649             if (my $tpm = $conf->{tpmstate0}) {
5650                 # start the TPM emulator so QEMU can connect on start
5651                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5652             }
5653
5654             my $exitcode = run_command($cmd, %run_params);
5655             if ($exitcode) {
5656                 if ($tpmpid) {
5657                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5658                     kill 'TERM', $tpmpid;
5659                 }
5660                 die "QEMU exited with code $exitcode\n";
5661             }
5662         };
5663     };
5664
5665     if ($conf->{hugepages}) {
5666
5667         my $code = sub {
5668             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5669             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5670
5671             PVE::QemuServer::Memory::hugepages_mount();
5672             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5673
5674             eval { $run_qemu->() };
5675             if (my $err = $@) {
5676                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5677                     if !$conf->{keephugepages};
5678                 die $err;
5679             }
5680
5681             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5682                 if !$conf->{keephugepages};
5683         };
5684         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5685
5686     } else {
5687         eval { $run_qemu->() };
5688     }
5689
5690     if (my $err = $@) {
5691         # deactivate volumes if start fails
5692         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5693         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5694
5695         die "start failed: $err";
5696     }
5697
5698     # re-reserve all PCI IDs now that we can know the actual VM PID
5699     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5700     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5701     warn $@ if $@;
5702
5703     print "migration listens on $migrate_uri\n" if $migrate_uri;
5704     $res->{migrate_uri} = $migrate_uri;
5705
5706     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5707         eval { mon_cmd($vmid, "cont"); };
5708         warn $@ if $@;
5709     }
5710
5711     #start nbd server for storage migration
5712     if (my $nbd = $migrate_opts->{nbd}) {
5713         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5714
5715         my $migrate_storage_uri;
5716         # nbd_protocol_version > 0 for unix socket support
5717         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5718             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5719             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5720             $migrate_storage_uri = "nbd:unix:$socket_path";
5721         } else {
5722             my $nodename = nodename();
5723             my $localip = $get_migration_ip->($nodename);
5724             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5725             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5726
5727             mon_cmd($vmid, "nbd-server-start", addr => {
5728                 type => 'inet',
5729                 data => {
5730                     host => "${localip}",
5731                     port => "${storage_migrate_port}",
5732                 },
5733             });
5734             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5735             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5736         }
5737
5738         $res->{migrate_storage_uri} = $migrate_storage_uri;
5739
5740         foreach my $opt (sort keys %$nbd) {
5741             my $drivestr = $nbd->{$opt}->{drivestr};
5742             my $volid = $nbd->{$opt}->{volid};
5743             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5744             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5745             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5746             print "re-using replicated volume: $opt - $volid\n"
5747                 if $nbd->{$opt}->{replicated};
5748
5749             $res->{drives}->{$opt} = $nbd->{$opt};
5750             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5751         }
5752     }
5753
5754     if ($migratedfrom) {
5755         eval {
5756             set_migration_caps($vmid);
5757         };
5758         warn $@ if $@;
5759
5760         if ($spice_port) {
5761             print "spice listens on port $spice_port\n";
5762             $res->{spice_port} = $spice_port;
5763             if ($migrate_opts->{spice_ticket}) {
5764                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5765                     $migrate_opts->{spice_ticket});
5766                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5767             }
5768         }
5769
5770     } else {
5771         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5772             if !$statefile && $conf->{balloon};
5773
5774         foreach my $opt (keys %$conf) {
5775             next if $opt !~  m/^net\d+$/;
5776             my $nicconf = parse_net($conf->{$opt});
5777             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5778         }
5779     }
5780
5781     mon_cmd($vmid, 'qom-set',
5782                 path => "machine/peripheral/balloon0",
5783                 property => "guest-stats-polling-interval",
5784                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5785
5786     if ($resume) {
5787         print "Resumed VM, removing state\n";
5788         if (my $vmstate = $conf->{vmstate}) {
5789             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5790             PVE::Storage::vdisk_free($storecfg, $vmstate);
5791         }
5792         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5793         PVE::QemuConfig->write_config($vmid, $conf);
5794     }
5795
5796     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5797
5798     return $res;
5799 }
5800
5801 sub vm_commandline {
5802     my ($storecfg, $vmid, $snapname) = @_;
5803
5804     my $conf = PVE::QemuConfig->load_config($vmid);
5805
5806     my ($forcemachine, $forcecpu);
5807     if ($snapname) {
5808         my $snapshot = $conf->{snapshots}->{$snapname};
5809         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5810
5811         # check for machine or CPU overrides in snapshot
5812         $forcemachine = $snapshot->{runningmachine};
5813         $forcecpu = $snapshot->{runningcpu};
5814
5815         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5816
5817         $conf = $snapshot;
5818     }
5819
5820     my $defaults = load_defaults();
5821
5822     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5823
5824     return PVE::Tools::cmd2string($cmd);
5825 }
5826
5827 sub vm_reset {
5828     my ($vmid, $skiplock) = @_;
5829
5830     PVE::QemuConfig->lock_config($vmid, sub {
5831
5832         my $conf = PVE::QemuConfig->load_config($vmid);
5833
5834         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5835
5836         mon_cmd($vmid, "system_reset");
5837     });
5838 }
5839
5840 sub get_vm_volumes {
5841     my ($conf) = @_;
5842
5843     my $vollist = [];
5844     foreach_volid($conf, sub {
5845         my ($volid, $attr) = @_;
5846
5847         return if $volid =~ m|^/|;
5848
5849         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5850         return if !$sid;
5851
5852         push @$vollist, $volid;
5853     });
5854
5855     return $vollist;
5856 }
5857
5858 sub vm_stop_cleanup {
5859     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5860
5861     eval {
5862
5863         if (!$keepActive) {
5864             my $vollist = get_vm_volumes($conf);
5865             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5866
5867             if (my $tpmdrive = $conf->{tpmstate0}) {
5868                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5869                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5870                 if ($storeid) {
5871                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5872                 }
5873             }
5874         }
5875
5876         foreach my $ext (qw(mon qmp pid vnc qga)) {
5877             unlink "/var/run/qemu-server/${vmid}.$ext";
5878         }
5879
5880         if ($conf->{ivshmem}) {
5881             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5882             # just delete it for now, VMs which have this already open do not
5883             # are affected, but new VMs will get a separated one. If this
5884             # becomes an issue we either add some sort of ref-counting or just
5885             # add a "don't delete on stop" flag to the ivshmem format.
5886             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5887         }
5888
5889         my $ids = [];
5890         foreach my $key (keys %$conf) {
5891             next if $key !~ m/^hostpci(\d+)$/;
5892             my $hostpciindex = $1;
5893             my $d = parse_hostpci($conf->{$key});
5894             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5895
5896             foreach my $pci (@{$d->{pciid}}) {
5897                 my $pciid = $pci->{id};
5898                 push @$ids, $pci->{id};
5899                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5900             }
5901         }
5902         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5903
5904         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5905     };
5906     warn $@ if $@; # avoid errors - just warn
5907 }
5908
5909 # call only in locked context
5910 sub _do_vm_stop {
5911     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5912
5913     my $pid = check_running($vmid, $nocheck);
5914     return if !$pid;
5915
5916     my $conf;
5917     if (!$nocheck) {
5918         $conf = PVE::QemuConfig->load_config($vmid);
5919         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5920         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5921             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5922             $timeout = $opts->{down} if $opts->{down};
5923         }
5924         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5925     }
5926
5927     eval {
5928         if ($shutdown) {
5929             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5930                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5931             } else {
5932                 mon_cmd($vmid, "system_powerdown");
5933             }
5934         } else {
5935             mon_cmd($vmid, "quit");
5936         }
5937     };
5938     my $err = $@;
5939
5940     if (!$err) {
5941         $timeout = 60 if !defined($timeout);
5942
5943         my $count = 0;
5944         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5945             $count++;
5946             sleep 1;
5947         }
5948
5949         if ($count >= $timeout) {
5950             if ($force) {
5951                 warn "VM still running - terminating now with SIGTERM\n";
5952                 kill 15, $pid;
5953             } else {
5954                 die "VM quit/powerdown failed - got timeout\n";
5955             }
5956         } else {
5957             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5958             return;
5959         }
5960     } else {
5961        if (!check_running($vmid, $nocheck)) {
5962             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5963             return;
5964        }
5965        if ($force) {
5966             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5967             kill 15, $pid;
5968         } else {
5969             die "VM quit/powerdown failed\n";
5970         }
5971     }
5972
5973     # wait again
5974     $timeout = 10;
5975
5976     my $count = 0;
5977     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5978         $count++;
5979         sleep 1;
5980     }
5981
5982     if ($count >= $timeout) {
5983         warn "VM still running - terminating now with SIGKILL\n";
5984         kill 9, $pid;
5985         sleep 1;
5986     }
5987
5988     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5989 }
5990
5991 # Note: use $nocheck to skip tests if VM configuration file exists.
5992 # We need that when migration VMs to other nodes (files already moved)
5993 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5994 sub vm_stop {
5995     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5996
5997     $force = 1 if !defined($force) && !$shutdown;
5998
5999     if ($migratedfrom){
6000         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6001         kill 15, $pid if $pid;
6002         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6003         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6004         return;
6005     }
6006
6007     PVE::QemuConfig->lock_config($vmid, sub {
6008         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6009    });
6010 }
6011
6012 sub vm_reboot {
6013     my ($vmid, $timeout) = @_;
6014
6015     PVE::QemuConfig->lock_config($vmid, sub {
6016         eval {
6017
6018             # only reboot if running, as qmeventd starts it again on a stop event
6019             return if !check_running($vmid);
6020
6021             create_reboot_request($vmid);
6022
6023             my $storecfg = PVE::Storage::config();
6024             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6025
6026         };
6027         if (my $err = $@) {
6028             # avoid that the next normal shutdown will be confused for a reboot
6029             clear_reboot_request($vmid);
6030             die $err;
6031         }
6032    });
6033 }
6034
6035 # note: if using the statestorage parameter, the caller has to check privileges
6036 sub vm_suspend {
6037     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6038
6039     my $conf;
6040     my $path;
6041     my $storecfg;
6042     my $vmstate;
6043
6044     PVE::QemuConfig->lock_config($vmid, sub {
6045
6046         $conf = PVE::QemuConfig->load_config($vmid);
6047
6048         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6049         PVE::QemuConfig->check_lock($conf)
6050             if !($skiplock || $is_backing_up);
6051
6052         die "cannot suspend to disk during backup\n"
6053             if $is_backing_up && $includestate;
6054
6055         if ($includestate) {
6056             $conf->{lock} = 'suspending';
6057             my $date = strftime("%Y-%m-%d", localtime(time()));
6058             $storecfg = PVE::Storage::config();
6059             if (!$statestorage) {
6060                 $statestorage = find_vmstate_storage($conf, $storecfg);
6061                 # check permissions for the storage
6062                 my $rpcenv = PVE::RPCEnvironment::get();
6063                 if ($rpcenv->{type} ne 'cli') {
6064                     my $authuser = $rpcenv->get_user();
6065                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6066                 }
6067             }
6068
6069
6070             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6071                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6072             $path = PVE::Storage::path($storecfg, $vmstate);
6073             PVE::QemuConfig->write_config($vmid, $conf);
6074         } else {
6075             mon_cmd($vmid, "stop");
6076         }
6077     });
6078
6079     if ($includestate) {
6080         # save vm state
6081         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6082
6083         eval {
6084             set_migration_caps($vmid, 1);
6085             mon_cmd($vmid, "savevm-start", statefile => $path);
6086             for(;;) {
6087                 my $state = mon_cmd($vmid, "query-savevm");
6088                 if (!$state->{status}) {
6089                     die "savevm not active\n";
6090                 } elsif ($state->{status} eq 'active') {
6091                     sleep(1);
6092                     next;
6093                 } elsif ($state->{status} eq 'completed') {
6094                     print "State saved, quitting\n";
6095                     last;
6096                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6097                     die "query-savevm failed with error '$state->{error}'\n"
6098                 } else {
6099                     die "query-savevm returned status '$state->{status}'\n";
6100                 }
6101             }
6102         };
6103         my $err = $@;
6104
6105         PVE::QemuConfig->lock_config($vmid, sub {
6106             $conf = PVE::QemuConfig->load_config($vmid);
6107             if ($err) {
6108                 # cleanup, but leave suspending lock, to indicate something went wrong
6109                 eval {
6110                     mon_cmd($vmid, "savevm-end");
6111                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6112                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6113                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6114                     PVE::QemuConfig->write_config($vmid, $conf);
6115                 };
6116                 warn $@ if $@;
6117                 die $err;
6118             }
6119
6120             die "lock changed unexpectedly\n"
6121                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6122
6123             mon_cmd($vmid, "quit");
6124             $conf->{lock} = 'suspended';
6125             PVE::QemuConfig->write_config($vmid, $conf);
6126         });
6127     }
6128 }
6129
6130 sub vm_resume {
6131     my ($vmid, $skiplock, $nocheck) = @_;
6132
6133     PVE::QemuConfig->lock_config($vmid, sub {
6134         my $res = mon_cmd($vmid, 'query-status');
6135         my $resume_cmd = 'cont';
6136         my $reset = 0;
6137
6138         if ($res->{status}) {
6139             return if $res->{status} eq 'running'; # job done, go home
6140             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6141             $reset = 1 if $res->{status} eq 'shutdown';
6142         }
6143
6144         if (!$nocheck) {
6145
6146             my $conf = PVE::QemuConfig->load_config($vmid);
6147
6148             PVE::QemuConfig->check_lock($conf)
6149                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6150         }
6151
6152         if ($reset) {
6153             # required if a VM shuts down during a backup and we get a resume
6154             # request before the backup finishes for example
6155             mon_cmd($vmid, "system_reset");
6156         }
6157         mon_cmd($vmid, $resume_cmd);
6158     });
6159 }
6160
6161 sub vm_sendkey {
6162     my ($vmid, $skiplock, $key) = @_;
6163
6164     PVE::QemuConfig->lock_config($vmid, sub {
6165
6166         my $conf = PVE::QemuConfig->load_config($vmid);
6167
6168         # there is no qmp command, so we use the human monitor command
6169         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6170         die $res if $res ne '';
6171     });
6172 }
6173
6174 # vzdump restore implementaion
6175
6176 sub tar_archive_read_firstfile {
6177     my $archive = shift;
6178
6179     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6180
6181     # try to detect archive type first
6182     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6183         die "unable to open file '$archive'\n";
6184     my $firstfile = <$fh>;
6185     kill 15, $pid;
6186     close $fh;
6187
6188     die "ERROR: archive contaions no data\n" if !$firstfile;
6189     chomp $firstfile;
6190
6191     return $firstfile;
6192 }
6193
6194 sub tar_restore_cleanup {
6195     my ($storecfg, $statfile) = @_;
6196
6197     print STDERR "starting cleanup\n";
6198
6199     if (my $fd = IO::File->new($statfile, "r")) {
6200         while (defined(my $line = <$fd>)) {
6201             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6202                 my $volid = $2;
6203                 eval {
6204                     if ($volid =~ m|^/|) {
6205                         unlink $volid || die 'unlink failed\n';
6206                     } else {
6207                         PVE::Storage::vdisk_free($storecfg, $volid);
6208                     }
6209                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6210                 };
6211                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6212             } else {
6213                 print STDERR "unable to parse line in statfile - $line";
6214             }
6215         }
6216         $fd->close();
6217     }
6218 }
6219
6220 sub restore_file_archive {
6221     my ($archive, $vmid, $user, $opts) = @_;
6222
6223     return restore_vma_archive($archive, $vmid, $user, $opts)
6224         if $archive eq '-';
6225
6226     my $info = PVE::Storage::archive_info($archive);
6227     my $format = $opts->{format} // $info->{format};
6228     my $comp = $info->{compression};
6229
6230     # try to detect archive format
6231     if ($format eq 'tar') {
6232         return restore_tar_archive($archive, $vmid, $user, $opts);
6233     } else {
6234         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6235     }
6236 }
6237
6238 # hepler to remove disks that will not be used after restore
6239 my $restore_cleanup_oldconf = sub {
6240     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6241
6242     my $kept_disks = {};
6243
6244     PVE::QemuConfig->foreach_volume($oldconf, sub {
6245         my ($ds, $drive) = @_;
6246
6247         return if drive_is_cdrom($drive, 1);
6248
6249         my $volid = $drive->{file};
6250         return if !$volid || $volid =~ m|^/|;
6251
6252         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6253         return if !$path || !$owner || ($owner != $vmid);
6254
6255         # Note: only delete disk we want to restore
6256         # other volumes will become unused
6257         if ($virtdev_hash->{$ds}) {
6258             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6259             if (my $err = $@) {
6260                 warn $err;
6261             }
6262         } else {
6263             $kept_disks->{$volid} = 1;
6264         }
6265     });
6266
6267     # after the restore we have no snapshots anymore
6268     for my $snapname (keys $oldconf->{snapshots}->%*) {
6269         my $snap = $oldconf->{snapshots}->{$snapname};
6270         if ($snap->{vmstate}) {
6271             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6272             if (my $err = $@) {
6273                 warn $err;
6274             }
6275         }
6276
6277         for my $volid (keys $kept_disks->%*) {
6278             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6279             warn $@ if $@;
6280         }
6281     }
6282 };
6283
6284 # Helper to parse vzdump backup device hints
6285 #
6286 # $rpcenv: Environment, used to ckeck storage permissions
6287 # $user: User ID, to check storage permissions
6288 # $storecfg: Storage configuration
6289 # $fh: the file handle for reading the configuration
6290 # $devinfo: should contain device sizes for all backu-up'ed devices
6291 # $options: backup options (pool, default storage)
6292 #
6293 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6294 my $parse_backup_hints = sub {
6295     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6296
6297     my $check_storage = sub { # assert if an image can be allocate
6298         my ($storeid, $scfg) = @_;
6299         die "Content type 'images' is not available on storage '$storeid'\n"
6300             if !$scfg->{content}->{images};
6301         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6302             if $user ne 'root@pam';
6303     };
6304
6305     my $virtdev_hash = {};
6306     while (defined(my $line = <$fh>)) {
6307         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6308             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6309             die "archive does not contain data for drive '$virtdev'\n"
6310                 if !$devinfo->{$devname};
6311
6312             if (defined($options->{storage})) {
6313                 $storeid = $options->{storage} || 'local';
6314             } elsif (!$storeid) {
6315                 $storeid = 'local';
6316             }
6317             $format = 'raw' if !$format;
6318             $devinfo->{$devname}->{devname} = $devname;
6319             $devinfo->{$devname}->{virtdev} = $virtdev;
6320             $devinfo->{$devname}->{format} = $format;
6321             $devinfo->{$devname}->{storeid} = $storeid;
6322
6323             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6324             $check_storage->($storeid, $scfg); # permission and content type check
6325
6326             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6327         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6328             my $virtdev = $1;
6329             my $drive = parse_drive($virtdev, $2);
6330
6331             if (drive_is_cloudinit($drive)) {
6332                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6333                 $storeid = $options->{storage} if defined ($options->{storage});
6334                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6335                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6336
6337                 $check_storage->($storeid, $scfg); # permission and content type check
6338
6339                 $virtdev_hash->{$virtdev} = {
6340                     format => $format,
6341                     storeid => $storeid,
6342                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6343                     is_cloudinit => 1,
6344                 };
6345             }
6346         }
6347     }
6348
6349     return $virtdev_hash;
6350 };
6351
6352 # Helper to allocate and activate all volumes required for a restore
6353 #
6354 # $storecfg: Storage configuration
6355 # $virtdev_hash: as returned by parse_backup_hints()
6356 #
6357 # Returns: { $virtdev => $volid }
6358 my $restore_allocate_devices = sub {
6359     my ($storecfg, $virtdev_hash, $vmid) = @_;
6360
6361     my $map = {};
6362     foreach my $virtdev (sort keys %$virtdev_hash) {
6363         my $d = $virtdev_hash->{$virtdev};
6364         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6365         my $storeid = $d->{storeid};
6366         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6367
6368         # test if requested format is supported
6369         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6370         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6371         $d->{format} = $defFormat if !$supported;
6372
6373         my $name;
6374         if ($d->{is_cloudinit}) {
6375             $name = "vm-$vmid-cloudinit";
6376             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6377             if ($scfg->{path}) {
6378                 $name .= ".$d->{format}";
6379             }
6380         }
6381
6382         my $volid = PVE::Storage::vdisk_alloc(
6383             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6384
6385         print STDERR "new volume ID is '$volid'\n";
6386         $d->{volid} = $volid;
6387
6388         PVE::Storage::activate_volumes($storecfg, [$volid]);
6389
6390         $map->{$virtdev} = $volid;
6391     }
6392
6393     return $map;
6394 };
6395
6396 sub restore_update_config_line {
6397     my ($cookie, $map, $line, $unique) = @_;
6398
6399     return '' if $line =~ m/^\#qmdump\#/;
6400     return '' if $line =~ m/^\#vzdump\#/;
6401     return '' if $line =~ m/^lock:/;
6402     return '' if $line =~ m/^unused\d+:/;
6403     return '' if $line =~ m/^parent:/;
6404
6405     my $res = '';
6406
6407     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6408     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6409         # try to convert old 1.X settings
6410         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6411         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6412             my ($model, $macaddr) = split(/\=/, $devconfig);
6413             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6414             my $net = {
6415                 model => $model,
6416                 bridge => "vmbr$ind",
6417                 macaddr => $macaddr,
6418             };
6419             my $netstr = print_net($net);
6420
6421             $res .= "net$cookie->{netcount}: $netstr\n";
6422             $cookie->{netcount}++;
6423         }
6424     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6425         my ($id, $netstr) = ($1, $2);
6426         my $net = parse_net($netstr);
6427         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6428         $netstr = print_net($net);
6429         $res .= "$id: $netstr\n";
6430     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6431         my $virtdev = $1;
6432         my $value = $3;
6433         my $di = parse_drive($virtdev, $value);
6434         if (defined($di->{backup}) && !$di->{backup}) {
6435             $res .= "#$line";
6436         } elsif ($map->{$virtdev}) {
6437             delete $di->{format}; # format can change on restore
6438             $di->{file} = $map->{$virtdev};
6439             $value = print_drive($di);
6440             $res .= "$virtdev: $value\n";
6441         } else {
6442             $res .= $line;
6443         }
6444     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6445         my $vmgenid = $1;
6446         if ($vmgenid ne '0') {
6447             # always generate a new vmgenid if there was a valid one setup
6448             $vmgenid = generate_uuid();
6449         }
6450         $res .= "vmgenid: $vmgenid\n";
6451     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6452         my ($uuid, $uuid_str);
6453         UUID::generate($uuid);
6454         UUID::unparse($uuid, $uuid_str);
6455         my $smbios1 = parse_smbios1($2);
6456         $smbios1->{uuid} = $uuid_str;
6457         $res .= $1.print_smbios1($smbios1)."\n";
6458     } else {
6459         $res .= $line;
6460     }
6461
6462     return $res;
6463 }
6464
6465 my $restore_deactivate_volumes = sub {
6466     my ($storecfg, $virtdev_hash) = @_;
6467
6468     my $vollist = [];
6469     for my $dev (values $virtdev_hash->%*) {
6470         push $vollist->@*, $dev->{volid} if $dev->{volid};
6471     }
6472
6473     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6474     print STDERR $@ if $@;
6475 };
6476
6477 my $restore_destroy_volumes = sub {
6478     my ($storecfg, $virtdev_hash) = @_;
6479
6480     for my $dev (values $virtdev_hash->%*) {
6481         my $volid = $dev->{volid} or next;
6482         eval {
6483             PVE::Storage::vdisk_free($storecfg, $volid);
6484             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6485         };
6486         print STDERR "unable to cleanup '$volid' - $@" if $@;
6487     }
6488 };
6489
6490 my $restore_merge_config = sub {
6491     my ($filename, $backup_conf_raw, $override_conf) = @_;
6492
6493     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6494     for my $key (keys $override_conf->%*) {
6495         $backup_conf->{$key} = $override_conf->{$key};
6496     }
6497
6498     return $backup_conf;
6499 };
6500
6501 sub scan_volids {
6502     my ($cfg, $vmid) = @_;
6503
6504     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6505
6506     my $volid_hash = {};
6507     foreach my $storeid (keys %$info) {
6508         foreach my $item (@{$info->{$storeid}}) {
6509             next if !($item->{volid} && $item->{size});
6510             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6511             $volid_hash->{$item->{volid}} = $item;
6512         }
6513     }
6514
6515     return $volid_hash;
6516 }
6517
6518 sub update_disk_config {
6519     my ($vmid, $conf, $volid_hash) = @_;
6520
6521     my $changes;
6522     my $prefix = "VM $vmid";
6523
6524     # used and unused disks
6525     my $referenced = {};
6526
6527     # Note: it is allowed to define multiple storages with same path (alias), so
6528     # we need to check both 'volid' and real 'path' (two different volid can point
6529     # to the same path).
6530
6531     my $referencedpath = {};
6532
6533     # update size info
6534     PVE::QemuConfig->foreach_volume($conf, sub {
6535         my ($opt, $drive) = @_;
6536
6537         my $volid = $drive->{file};
6538         return if !$volid;
6539         my $volume = $volid_hash->{$volid};
6540
6541         # mark volid as "in-use" for next step
6542         $referenced->{$volid} = 1;
6543         if ($volume && (my $path = $volume->{path})) {
6544             $referencedpath->{$path} = 1;
6545         }
6546
6547         return if drive_is_cdrom($drive);
6548         return if !$volume;
6549
6550         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6551         if (defined($updated)) {
6552             $changes = 1;
6553             $conf->{$opt} = print_drive($updated);
6554             print "$prefix ($opt): $msg\n";
6555         }
6556     });
6557
6558     # remove 'unusedX' entry if volume is used
6559     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6560         my ($opt, $drive) = @_;
6561
6562         my $volid = $drive->{file};
6563         return if !$volid;
6564
6565         my $path;
6566         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6567         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6568             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6569             $changes = 1;
6570             delete $conf->{$opt};
6571         }
6572
6573         $referenced->{$volid} = 1;
6574         $referencedpath->{$path} = 1 if $path;
6575     });
6576
6577     foreach my $volid (sort keys %$volid_hash) {
6578         next if $volid =~ m/vm-$vmid-state-/;
6579         next if $referenced->{$volid};
6580         my $path = $volid_hash->{$volid}->{path};
6581         next if !$path; # just to be sure
6582         next if $referencedpath->{$path};
6583         $changes = 1;
6584         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6585         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6586         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6587     }
6588
6589     return $changes;
6590 }
6591
6592 sub rescan {
6593     my ($vmid, $nolock, $dryrun) = @_;
6594
6595     my $cfg = PVE::Storage::config();
6596
6597     print "rescan volumes...\n";
6598     my $volid_hash = scan_volids($cfg, $vmid);
6599
6600     my $updatefn =  sub {
6601         my ($vmid) = @_;
6602
6603         my $conf = PVE::QemuConfig->load_config($vmid);
6604
6605         PVE::QemuConfig->check_lock($conf);
6606
6607         my $vm_volids = {};
6608         foreach my $volid (keys %$volid_hash) {
6609             my $info = $volid_hash->{$volid};
6610             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6611         }
6612
6613         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6614
6615         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6616     };
6617
6618     if (defined($vmid)) {
6619         if ($nolock) {
6620             &$updatefn($vmid);
6621         } else {
6622             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6623         }
6624     } else {
6625         my $vmlist = config_list();
6626         foreach my $vmid (keys %$vmlist) {
6627             if ($nolock) {
6628                 &$updatefn($vmid);
6629             } else {
6630                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6631             }
6632         }
6633     }
6634 }
6635
6636 sub restore_proxmox_backup_archive {
6637     my ($archive, $vmid, $user, $options) = @_;
6638
6639     my $storecfg = PVE::Storage::config();
6640
6641     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6642     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6643
6644     my $fingerprint = $scfg->{fingerprint};
6645     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6646
6647     my $repo = PVE::PBSClient::get_repository($scfg);
6648     my $namespace = $scfg->{namespace};
6649
6650     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6651     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6652     local $ENV{PBS_PASSWORD} = $password;
6653     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6654
6655     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6656         PVE::Storage::parse_volname($storecfg, $archive);
6657
6658     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6659
6660     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6661
6662     my $tmpdir = "/var/tmp/vzdumptmp$$";
6663     rmtree $tmpdir;
6664     mkpath $tmpdir;
6665
6666     my $conffile = PVE::QemuConfig->config_file($vmid);
6667      # disable interrupts (always do cleanups)
6668     local $SIG{INT} =
6669         local $SIG{TERM} =
6670         local $SIG{QUIT} =
6671         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6672
6673     # Note: $oldconf is undef if VM does not exists
6674     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6675     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6676     my $new_conf_raw = '';
6677
6678     my $rpcenv = PVE::RPCEnvironment::get();
6679     my $devinfo = {}; # info about drives included in backup
6680     my $virtdev_hash = {}; # info about allocated drives
6681
6682     eval {
6683         # enable interrupts
6684         local $SIG{INT} =
6685             local $SIG{TERM} =
6686             local $SIG{QUIT} =
6687             local $SIG{HUP} =
6688             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6689
6690         my $cfgfn = "$tmpdir/qemu-server.conf";
6691         my $firewall_config_fn = "$tmpdir/fw.conf";
6692         my $index_fn = "$tmpdir/index.json";
6693
6694         my $cmd = "restore";
6695
6696         my $param = [$pbs_backup_name, "index.json", $index_fn];
6697         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6698         my $index = PVE::Tools::file_get_contents($index_fn);
6699         $index = decode_json($index);
6700
6701         foreach my $info (@{$index->{files}}) {
6702             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6703                 my $devname = $1;
6704                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6705                     $devinfo->{$devname}->{size} = $1;
6706                 } else {
6707                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6708                 }
6709             }
6710         }
6711
6712         my $is_qemu_server_backup = scalar(
6713             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6714         );
6715         if (!$is_qemu_server_backup) {
6716             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6717         }
6718         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6719
6720         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6721         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6722
6723         if ($has_firewall_config) {
6724             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6725             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6726
6727             my $pve_firewall_dir = '/etc/pve/firewall';
6728             mkdir $pve_firewall_dir; # make sure the dir exists
6729             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6730         }
6731
6732         my $fh = IO::File->new($cfgfn, "r") ||
6733             die "unable to read qemu-server.conf - $!\n";
6734
6735         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6736
6737         # fixme: rate limit?
6738
6739         # create empty/temp config
6740         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6741
6742         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6743
6744         # allocate volumes
6745         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6746
6747         foreach my $virtdev (sort keys %$virtdev_hash) {
6748             my $d = $virtdev_hash->{$virtdev};
6749             next if $d->{is_cloudinit}; # no need to restore cloudinit
6750
6751             # this fails if storage is unavailable
6752             my $volid = $d->{volid};
6753             my $path = PVE::Storage::path($storecfg, $volid);
6754
6755             # for live-restore we only want to preload the efidisk and TPM state
6756             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6757
6758             my @ns_arg;
6759             if (defined(my $ns = $scfg->{namespace})) {
6760                 @ns_arg = ('--ns', $ns);
6761             }
6762
6763             my $pbs_restore_cmd = [
6764                 '/usr/bin/pbs-restore',
6765                 '--repository', $repo,
6766                 @ns_arg,
6767                 $pbs_backup_name,
6768                 "$d->{devname}.img.fidx",
6769                 $path,
6770                 '--verbose',
6771                 ];
6772
6773             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6774             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6775
6776             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6777                 push @$pbs_restore_cmd, '--skip-zero';
6778             }
6779
6780             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6781             print "restore proxmox backup image: $dbg_cmdstring\n";
6782             run_command($pbs_restore_cmd);
6783         }
6784
6785         $fh->seek(0, 0) || die "seek failed - $!\n";
6786
6787         my $cookie = { netcount => 0 };
6788         while (defined(my $line = <$fh>)) {
6789             $new_conf_raw .= restore_update_config_line(
6790                 $cookie,
6791                 $map,
6792                 $line,
6793                 $options->{unique},
6794             );
6795         }
6796
6797         $fh->close();
6798     };
6799     my $err = $@;
6800
6801     if ($err || !$options->{live}) {
6802         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6803     }
6804
6805     rmtree $tmpdir;
6806
6807     if ($err) {
6808         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6809         die $err;
6810     }
6811
6812     if ($options->{live}) {
6813         # keep lock during live-restore
6814         $new_conf_raw .= "\nlock: create";
6815     }
6816
6817     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6818     PVE::QemuConfig->write_config($vmid, $new_conf);
6819
6820     eval { rescan($vmid, 1); };
6821     warn $@ if $@;
6822
6823     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6824
6825     if ($options->{live}) {
6826         # enable interrupts
6827         local $SIG{INT} =
6828             local $SIG{TERM} =
6829             local $SIG{QUIT} =
6830             local $SIG{HUP} =
6831             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6832
6833         my $conf = PVE::QemuConfig->load_config($vmid);
6834         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6835
6836         # these special drives are already restored before start
6837         delete $devinfo->{'drive-efidisk0'};
6838         delete $devinfo->{'drive-tpmstate0-backup'};
6839
6840         my $pbs_opts = {
6841             repo => $repo,
6842             keyfile => $keyfile,
6843             snapshot => $pbs_backup_name,
6844             namespace => $namespace,
6845         };
6846         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
6847
6848         PVE::QemuConfig->remove_lock($vmid, "create");
6849     }
6850 }
6851
6852 sub pbs_live_restore {
6853     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
6854
6855     print "starting VM for live-restore\n";
6856     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
6857
6858     my $pbs_backing = {};
6859     for my $ds (keys %$restored_disks) {
6860         $ds =~ m/^drive-(.*)$/;
6861         my $confname = $1;
6862         $pbs_backing->{$confname} = {
6863             repository => $opts->{repo},
6864             snapshot => $opts->{snapshot},
6865             archive => "$ds.img.fidx",
6866         };
6867         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
6868         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
6869
6870         my $drive = parse_drive($confname, $conf->{$confname});
6871         print "restoring '$ds' to '$drive->{file}'\n";
6872     }
6873
6874     my $drives_streamed = 0;
6875     eval {
6876         # make sure HA doesn't interrupt our restore by stopping the VM
6877         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6878             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6879         }
6880
6881         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6882         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6883         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6884
6885         my $qmeventd_fd = register_qmeventd_handle($vmid);
6886
6887         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6888         # this will effectively collapse the backing image chain consisting of
6889         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6890         # removes itself once all backing images vanish with 'auto-remove=on')
6891         my $jobs = {};
6892         for my $ds (sort keys %$restored_disks) {
6893             my $job_id = "restore-$ds";
6894             mon_cmd($vmid, 'block-stream',
6895                 'job-id' => $job_id,
6896                 device => "$ds",
6897             );
6898             $jobs->{$job_id} = {};
6899         }
6900
6901         mon_cmd($vmid, 'cont');
6902         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6903
6904         print "restore-drive jobs finished successfully, removing all tracking block devices"
6905             ." to disconnect from Proxmox Backup Server\n";
6906
6907         for my $ds (sort keys %$restored_disks) {
6908             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6909         }
6910
6911         close($qmeventd_fd);
6912     };
6913
6914     my $err = $@;
6915
6916     if ($err) {
6917         warn "An error occurred during live-restore: $err\n";
6918         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6919         die "live-restore failed\n";
6920     }
6921 }
6922
6923 sub restore_vma_archive {
6924     my ($archive, $vmid, $user, $opts, $comp) = @_;
6925
6926     my $readfrom = $archive;
6927
6928     my $cfg = PVE::Storage::config();
6929     my $commands = [];
6930     my $bwlimit = $opts->{bwlimit};
6931
6932     my $dbg_cmdstring = '';
6933     my $add_pipe = sub {
6934         my ($cmd) = @_;
6935         push @$commands, $cmd;
6936         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6937         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6938         $readfrom = '-';
6939     };
6940
6941     my $input = undef;
6942     if ($archive eq '-') {
6943         $input = '<&STDIN';
6944     } else {
6945         # If we use a backup from a PVE defined storage we also consider that
6946         # storage's rate limit:
6947         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6948         if (defined($volid)) {
6949             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6950             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6951             if ($readlimit) {
6952                 print STDERR "applying read rate limit: $readlimit\n";
6953                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6954                 $add_pipe->($cstream);
6955             }
6956         }
6957     }
6958
6959     if ($comp) {
6960         my $info = PVE::Storage::decompressor_info('vma', $comp);
6961         my $cmd = $info->{decompressor};
6962         push @$cmd, $readfrom;
6963         $add_pipe->($cmd);
6964     }
6965
6966     my $tmpdir = "/var/tmp/vzdumptmp$$";
6967     rmtree $tmpdir;
6968
6969     # disable interrupts (always do cleanups)
6970     local $SIG{INT} =
6971         local $SIG{TERM} =
6972         local $SIG{QUIT} =
6973         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6974
6975     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6976     POSIX::mkfifo($mapfifo, 0600);
6977     my $fifofh;
6978     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6979
6980     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6981
6982     my $oldtimeout;
6983     my $timeout = 5;
6984
6985     my $devinfo = {}; # info about drives included in backup
6986     my $virtdev_hash = {}; # info about allocated drives
6987
6988     my $rpcenv = PVE::RPCEnvironment::get();
6989
6990     my $conffile = PVE::QemuConfig->config_file($vmid);
6991
6992     # Note: $oldconf is undef if VM does not exist
6993     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6994     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6995     my $new_conf_raw = '';
6996
6997     my %storage_limits;
6998
6999     my $print_devmap = sub {
7000         my $cfgfn = "$tmpdir/qemu-server.conf";
7001
7002         # we can read the config - that is already extracted
7003         my $fh = IO::File->new($cfgfn, "r") ||
7004             die "unable to read qemu-server.conf - $!\n";
7005
7006         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7007         if (-f $fwcfgfn) {
7008             my $pve_firewall_dir = '/etc/pve/firewall';
7009             mkdir $pve_firewall_dir; # make sure the dir exists
7010             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7011         }
7012
7013         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7014
7015         foreach my $info (values %{$virtdev_hash}) {
7016             my $storeid = $info->{storeid};
7017             next if defined($storage_limits{$storeid});
7018
7019             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7020             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7021             $storage_limits{$storeid} = $limit * 1024;
7022         }
7023
7024         foreach my $devname (keys %$devinfo) {
7025             die "found no device mapping information for device '$devname'\n"
7026                 if !$devinfo->{$devname}->{virtdev};
7027         }
7028
7029         # create empty/temp config
7030         if ($oldconf) {
7031             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7032             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7033         }
7034
7035         # allocate volumes
7036         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7037
7038         # print restore information to $fifofh
7039         foreach my $virtdev (sort keys %$virtdev_hash) {
7040             my $d = $virtdev_hash->{$virtdev};
7041             next if $d->{is_cloudinit}; # no need to restore cloudinit
7042
7043             my $storeid = $d->{storeid};
7044             my $volid = $d->{volid};
7045
7046             my $map_opts = '';
7047             if (my $limit = $storage_limits{$storeid}) {
7048                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7049             }
7050
7051             my $write_zeros = 1;
7052             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7053                 $write_zeros = 0;
7054             }
7055
7056             my $path = PVE::Storage::path($cfg, $volid);
7057
7058             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7059
7060             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7061         }
7062
7063         $fh->seek(0, 0) || die "seek failed - $!\n";
7064
7065         my $cookie = { netcount => 0 };
7066         while (defined(my $line = <$fh>)) {
7067             $new_conf_raw .= restore_update_config_line(
7068                 $cookie,
7069                 $map,
7070                 $line,
7071                 $opts->{unique},
7072             );
7073         }
7074
7075         $fh->close();
7076     };
7077
7078     eval {
7079         # enable interrupts
7080         local $SIG{INT} =
7081             local $SIG{TERM} =
7082             local $SIG{QUIT} =
7083             local $SIG{HUP} =
7084             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7085         local $SIG{ALRM} = sub { die "got timeout\n"; };
7086
7087         $oldtimeout = alarm($timeout);
7088
7089         my $parser = sub {
7090             my $line = shift;
7091
7092             print "$line\n";
7093
7094             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7095                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7096                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7097             } elsif ($line =~ m/^CTIME: /) {
7098                 # we correctly received the vma config, so we can disable
7099                 # the timeout now for disk allocation (set to 10 minutes, so
7100                 # that we always timeout if something goes wrong)
7101                 alarm(600);
7102                 &$print_devmap();
7103                 print $fifofh "done\n";
7104                 my $tmp = $oldtimeout || 0;
7105                 $oldtimeout = undef;
7106                 alarm($tmp);
7107                 close($fifofh);
7108                 $fifofh = undef;
7109             }
7110         };
7111
7112         print "restore vma archive: $dbg_cmdstring\n";
7113         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7114     };
7115     my $err = $@;
7116
7117     alarm($oldtimeout) if $oldtimeout;
7118
7119     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7120
7121     close($fifofh) if $fifofh;
7122     unlink $mapfifo;
7123     rmtree $tmpdir;
7124
7125     if ($err) {
7126         $restore_destroy_volumes->($cfg, $virtdev_hash);
7127         die $err;
7128     }
7129
7130     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7131     PVE::QemuConfig->write_config($vmid, $new_conf);
7132
7133     eval { rescan($vmid, 1); };
7134     warn $@ if $@;
7135
7136     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7137 }
7138
7139 sub restore_tar_archive {
7140     my ($archive, $vmid, $user, $opts) = @_;
7141
7142     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7143         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7144         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7145     }
7146
7147     if ($archive ne '-') {
7148         my $firstfile = tar_archive_read_firstfile($archive);
7149         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7150             if $firstfile ne 'qemu-server.conf';
7151     }
7152
7153     my $storecfg = PVE::Storage::config();
7154
7155     # avoid zombie disks when restoring over an existing VM -> cleanup first
7156     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7157     # skiplock=1 because qmrestore has set the 'create' lock itself already
7158     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7159     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7160
7161     my $tocmd = "/usr/lib/qemu-server/qmextract";
7162
7163     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7164     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7165     $tocmd .= ' --prealloc' if $opts->{prealloc};
7166     $tocmd .= ' --info' if $opts->{info};
7167
7168     # tar option "xf" does not autodetect compression when read from STDIN,
7169     # so we pipe to zcat
7170     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7171         PVE::Tools::shellquote("--to-command=$tocmd");
7172
7173     my $tmpdir = "/var/tmp/vzdumptmp$$";
7174     mkpath $tmpdir;
7175
7176     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7177     local $ENV{VZDUMP_VMID} = $vmid;
7178     local $ENV{VZDUMP_USER} = $user;
7179
7180     my $conffile = PVE::QemuConfig->config_file($vmid);
7181     my $new_conf_raw = '';
7182
7183     # disable interrupts (always do cleanups)
7184     local $SIG{INT} =
7185         local $SIG{TERM} =
7186         local $SIG{QUIT} =
7187         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7188
7189     eval {
7190         # enable interrupts
7191         local $SIG{INT} =
7192             local $SIG{TERM} =
7193             local $SIG{QUIT} =
7194             local $SIG{HUP} =
7195             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7196
7197         if ($archive eq '-') {
7198             print "extracting archive from STDIN\n";
7199             run_command($cmd, input => "<&STDIN");
7200         } else {
7201             print "extracting archive '$archive'\n";
7202             run_command($cmd);
7203         }
7204
7205         return if $opts->{info};
7206
7207         # read new mapping
7208         my $map = {};
7209         my $statfile = "$tmpdir/qmrestore.stat";
7210         if (my $fd = IO::File->new($statfile, "r")) {
7211             while (defined (my $line = <$fd>)) {
7212                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7213                     $map->{$1} = $2 if $1;
7214                 } else {
7215                     print STDERR "unable to parse line in statfile - $line\n";
7216                 }
7217             }
7218             $fd->close();
7219         }
7220
7221         my $confsrc = "$tmpdir/qemu-server.conf";
7222
7223         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7224
7225         my $cookie = { netcount => 0 };
7226         while (defined (my $line = <$srcfd>)) {
7227             $new_conf_raw .= restore_update_config_line(
7228                 $cookie,
7229                 $map,
7230                 $line,
7231                 $opts->{unique},
7232             );
7233         }
7234
7235         $srcfd->close();
7236     };
7237     if (my $err = $@) {
7238         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7239         die $err;
7240     }
7241
7242     rmtree $tmpdir;
7243
7244     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7245
7246     PVE::Cluster::cfs_update(); # make sure we read new file
7247
7248     eval { rescan($vmid, 1); };
7249     warn $@ if $@;
7250 };
7251
7252 sub foreach_storage_used_by_vm {
7253     my ($conf, $func) = @_;
7254
7255     my $sidhash = {};
7256
7257     PVE::QemuConfig->foreach_volume($conf, sub {
7258         my ($ds, $drive) = @_;
7259         return if drive_is_cdrom($drive);
7260
7261         my $volid = $drive->{file};
7262
7263         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7264         $sidhash->{$sid} = $sid if $sid;
7265     });
7266
7267     foreach my $sid (sort keys %$sidhash) {
7268         &$func($sid);
7269     }
7270 }
7271
7272 my $qemu_snap_storage = {
7273     rbd => 1,
7274 };
7275 sub do_snapshots_with_qemu {
7276     my ($storecfg, $volid, $deviceid) = @_;
7277
7278     return if $deviceid =~ m/tpmstate0/;
7279
7280     my $storage_name = PVE::Storage::parse_volume_id($volid);
7281     my $scfg = $storecfg->{ids}->{$storage_name};
7282     die "could not find storage '$storage_name'\n" if !defined($scfg);
7283
7284     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7285         return 1;
7286     }
7287
7288     if ($volid =~ m/\.(qcow2|qed)$/){
7289         return 1;
7290     }
7291
7292     return;
7293 }
7294
7295 sub qga_check_running {
7296     my ($vmid, $nowarn) = @_;
7297
7298     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7299     if ($@) {
7300         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7301         return 0;
7302     }
7303     return 1;
7304 }
7305
7306 sub template_create {
7307     my ($vmid, $conf, $disk) = @_;
7308
7309     my $storecfg = PVE::Storage::config();
7310
7311     PVE::QemuConfig->foreach_volume($conf, sub {
7312         my ($ds, $drive) = @_;
7313
7314         return if drive_is_cdrom($drive);
7315         return if $disk && $ds ne $disk;
7316
7317         my $volid = $drive->{file};
7318         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7319
7320         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7321         $drive->{file} = $voliddst;
7322         $conf->{$ds} = print_drive($drive);
7323         PVE::QemuConfig->write_config($vmid, $conf);
7324     });
7325 }
7326
7327 sub convert_iscsi_path {
7328     my ($path) = @_;
7329
7330     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7331         my $portal = $1;
7332         my $target = $2;
7333         my $lun = $3;
7334
7335         my $initiator_name = get_initiator_name();
7336
7337         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7338                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7339     }
7340
7341     die "cannot convert iscsi path '$path', unkown format\n";
7342 }
7343
7344 sub qemu_img_convert {
7345     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7346
7347     my $storecfg = PVE::Storage::config();
7348     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7349     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7350
7351     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7352
7353     my $cachemode;
7354     my $src_path;
7355     my $src_is_iscsi = 0;
7356     my $src_format;
7357
7358     if ($src_storeid) {
7359         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7360         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7361         $src_format = qemu_img_format($src_scfg, $src_volname);
7362         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7363         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7364         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7365     } elsif (-f $src_volid || -b $src_volid) {
7366         $src_path = $src_volid;
7367         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7368             $src_format = $1;
7369         }
7370     }
7371
7372     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7373
7374     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7375     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7376     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7377     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7378
7379     my $cmd = [];
7380     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7381     push @$cmd, '-l', "snapshot.name=$snapname"
7382         if $snapname && $src_format && $src_format eq "qcow2";
7383     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7384     push @$cmd, '-T', $cachemode if defined($cachemode);
7385
7386     if ($src_is_iscsi) {
7387         push @$cmd, '--image-opts';
7388         $src_path = convert_iscsi_path($src_path);
7389     } elsif ($src_format) {
7390         push @$cmd, '-f', $src_format;
7391     }
7392
7393     if ($dst_is_iscsi) {
7394         push @$cmd, '--target-image-opts';
7395         $dst_path = convert_iscsi_path($dst_path);
7396     } else {
7397         push @$cmd, '-O', $dst_format;
7398     }
7399
7400     push @$cmd, $src_path;
7401
7402     if (!$dst_is_iscsi && $is_zero_initialized) {
7403         push @$cmd, "zeroinit:$dst_path";
7404     } else {
7405         push @$cmd, $dst_path;
7406     }
7407
7408     my $parser = sub {
7409         my $line = shift;
7410         if($line =~ m/\((\S+)\/100\%\)/){
7411             my $percent = $1;
7412             my $transferred = int($size * $percent / 100);
7413             my $total_h = render_bytes($size, 1);
7414             my $transferred_h = render_bytes($transferred, 1);
7415
7416             print "transferred $transferred_h of $total_h ($percent%)\n";
7417         }
7418
7419     };
7420
7421     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7422     my $err = $@;
7423     die "copy failed: $err" if $err;
7424 }
7425
7426 sub qemu_img_format {
7427     my ($scfg, $volname) = @_;
7428
7429     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7430         return $1;
7431     } else {
7432         return "raw";
7433     }
7434 }
7435
7436 sub qemu_drive_mirror {
7437     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7438
7439     $jobs = {} if !$jobs;
7440
7441     my $qemu_target;
7442     my $format;
7443     $jobs->{"drive-$drive"} = {};
7444
7445     if ($dst_volid =~ /^nbd:/) {
7446         $qemu_target = $dst_volid;
7447         $format = "nbd";
7448     } else {
7449         my $storecfg = PVE::Storage::config();
7450         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7451
7452         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7453
7454         $format = qemu_img_format($dst_scfg, $dst_volname);
7455
7456         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7457
7458         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7459     }
7460
7461     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7462     $opts->{format} = $format if $format;
7463
7464     if (defined($src_bitmap)) {
7465         $opts->{sync} = 'incremental';
7466         $opts->{bitmap} = $src_bitmap;
7467         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7468     }
7469
7470     if (defined($bwlimit)) {
7471         $opts->{speed} = $bwlimit * 1024;
7472         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7473     } else {
7474         print "drive mirror is starting for drive-$drive\n";
7475     }
7476
7477     # if a job already runs for this device we get an error, catch it for cleanup
7478     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7479     if (my $err = $@) {
7480         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7481         warn "$@\n" if $@;
7482         die "mirroring error: $err\n";
7483     }
7484
7485     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7486 }
7487
7488 # $completion can be either
7489 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7490 # 'cancel': wait until all jobs are ready, block-job-cancel them
7491 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7492 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7493 sub qemu_drive_mirror_monitor {
7494     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7495
7496     $completion //= 'complete';
7497     $op //= "mirror";
7498
7499     eval {
7500         my $err_complete = 0;
7501
7502         my $starttime = time ();
7503         while (1) {
7504             die "block job ('$op') timed out\n" if $err_complete > 300;
7505
7506             my $stats = mon_cmd($vmid, "query-block-jobs");
7507             my $ctime = time();
7508
7509             my $running_jobs = {};
7510             for my $stat (@$stats) {
7511                 next if $stat->{type} ne $op;
7512                 $running_jobs->{$stat->{device}} = $stat;
7513             }
7514
7515             my $readycounter = 0;
7516
7517             for my $job_id (sort keys %$jobs) {
7518                 my $job = $running_jobs->{$job_id};
7519
7520                 my $vanished = !defined($job);
7521                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7522                 if($complete || ($vanished && $completion eq 'auto')) {
7523                     print "$job_id: $op-job finished\n";
7524                     delete $jobs->{$job_id};
7525                     next;
7526                 }
7527
7528                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7529
7530                 my $busy = $job->{busy};
7531                 my $ready = $job->{ready};
7532                 if (my $total = $job->{len}) {
7533                     my $transferred = $job->{offset} || 0;
7534                     my $remaining = $total - $transferred;
7535                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7536
7537                     my $duration = $ctime - $starttime;
7538                     my $total_h = render_bytes($total, 1);
7539                     my $transferred_h = render_bytes($transferred, 1);
7540
7541                     my $status = sprintf(
7542                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7543                         render_duration($duration),
7544                     );
7545
7546                     if ($ready) {
7547                         if ($busy) {
7548                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7549                         } else {
7550                             $status .= ", ready";
7551                         }
7552                     }
7553                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7554                     $jobs->{$job_id}->{ready} = $ready;
7555                 }
7556
7557                 $readycounter++ if $job->{ready};
7558             }
7559
7560             last if scalar(keys %$jobs) == 0;
7561
7562             if ($readycounter == scalar(keys %$jobs)) {
7563                 print "all '$op' jobs are ready\n";
7564
7565                 # do the complete later (or has already been done)
7566                 last if $completion eq 'skip' || $completion eq 'auto';
7567
7568                 if ($vmiddst && $vmiddst != $vmid) {
7569                     my $agent_running = $qga && qga_check_running($vmid);
7570                     if ($agent_running) {
7571                         print "freeze filesystem\n";
7572                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7573                         warn $@ if $@;
7574                     } else {
7575                         print "suspend vm\n";
7576                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7577                         warn $@ if $@;
7578                     }
7579
7580                     # if we clone a disk for a new target vm, we don't switch the disk
7581                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7582
7583                     if ($agent_running) {
7584                         print "unfreeze filesystem\n";
7585                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7586                         warn $@ if $@;
7587                     } else {
7588                         print "resume vm\n";
7589                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7590                         warn $@ if $@;
7591                     }
7592
7593                     last;
7594                 } else {
7595
7596                     for my $job_id (sort keys %$jobs) {
7597                         # try to switch the disk if source and destination are on the same guest
7598                         print "$job_id: Completing block job_id...\n";
7599
7600                         my $op;
7601                         if ($completion eq 'complete') {
7602                             $op = 'block-job-complete';
7603                         } elsif ($completion eq 'cancel') {
7604                             $op = 'block-job-cancel';
7605                         } else {
7606                             die "invalid completion value: $completion\n";
7607                         }
7608                         eval { mon_cmd($vmid, $op, device => $job_id) };
7609                         if ($@ =~ m/cannot be completed/) {
7610                             print "$job_id: block job cannot be completed, trying again.\n";
7611                             $err_complete++;
7612                         }else {
7613                             print "$job_id: Completed successfully.\n";
7614                             $jobs->{$job_id}->{complete} = 1;
7615                         }
7616                     }
7617                 }
7618             }
7619             sleep 1;
7620         }
7621     };
7622     my $err = $@;
7623
7624     if ($err) {
7625         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7626         die "block job ($op) error: $err";
7627     }
7628 }
7629
7630 sub qemu_blockjobs_cancel {
7631     my ($vmid, $jobs) = @_;
7632
7633     foreach my $job (keys %$jobs) {
7634         print "$job: Cancelling block job\n";
7635         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7636         $jobs->{$job}->{cancel} = 1;
7637     }
7638
7639     while (1) {
7640         my $stats = mon_cmd($vmid, "query-block-jobs");
7641
7642         my $running_jobs = {};
7643         foreach my $stat (@$stats) {
7644             $running_jobs->{$stat->{device}} = $stat;
7645         }
7646
7647         foreach my $job (keys %$jobs) {
7648
7649             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7650                 print "$job: Done.\n";
7651                 delete $jobs->{$job};
7652             }
7653         }
7654
7655         last if scalar(keys %$jobs) == 0;
7656
7657         sleep 1;
7658     }
7659 }
7660
7661 sub clone_disk {
7662     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7663
7664     my ($vmid, $running) = $source->@{qw(vmid running)};
7665     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7666
7667     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7668     my ($storage, $format) = $dest->@{qw(storage format)};
7669
7670     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7671
7672     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7673         die "cloning from/to EFI disk requires EFI disk\n"
7674             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7675         die "cloning from/to TPM state requires TPM state\n"
7676             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7677
7678         # This would lead to two device nodes in QEMU pointing to the same backing image!
7679         die "cannot change drive name when cloning disk from/to the same VM\n"
7680             if $use_drive_mirror && $vmid == $newvmid;
7681     }
7682
7683     die "cannot move TPM state while VM is running\n"
7684         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7685
7686     my $newvolid;
7687
7688     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7689     print "$src_drivename " if $src_drivename;
7690     print "($drive->{file})\n";
7691
7692     if (!$full) {
7693         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7694         push @$newvollist, $newvolid;
7695     } else {
7696
7697         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7698         $storeid = $storage if $storage;
7699
7700         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7701
7702         my $name = undef;
7703         my $size = undef;
7704         if (drive_is_cloudinit($drive)) {
7705             $name = "vm-$newvmid-cloudinit";
7706             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7707             if ($scfg->{path}) {
7708                 $name .= ".$dst_format";
7709             }
7710             $snapname = undef;
7711             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7712         } elsif ($dst_drivename eq 'efidisk0') {
7713             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7714         } elsif ($dst_drivename eq 'tpmstate0') {
7715             $dst_format = 'raw';
7716             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7717         } else {
7718             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7719         }
7720         $newvolid = PVE::Storage::vdisk_alloc(
7721             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7722         );
7723         push @$newvollist, $newvolid;
7724
7725         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7726
7727         if (drive_is_cloudinit($drive)) {
7728             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7729             # if this is the case, we have to complete any block-jobs still there from
7730             # previous drive-mirrors
7731             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7732                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7733             }
7734             goto no_data_clone;
7735         }
7736
7737         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7738         if ($use_drive_mirror) {
7739             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7740                 $completion, $qga, $bwlimit);
7741         } else {
7742             # TODO: handle bwlimits
7743             if ($dst_drivename eq 'efidisk0') {
7744                 # the relevant data on the efidisk may be smaller than the source
7745                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7746                 # that is given by the OVMF_VARS.fd
7747                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7748                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7749
7750                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7751
7752                 # better for Ceph if block size is not too small, see bug #3324
7753                 my $bs = 1024*1024;
7754
7755                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7756
7757                 if ($src_format eq 'qcow2' && $snapname) {
7758                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7759                         if !min_version(kvm_user_version(), 6, 2);
7760                     push $cmd->@*, '-l', $snapname;
7761                 }
7762                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7763                 run_command($cmd);
7764             } else {
7765                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7766             }
7767         }
7768     }
7769
7770 no_data_clone:
7771     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7772
7773     my $disk = dclone($drive);
7774     delete $disk->{format};
7775     $disk->{file} = $newvolid;
7776     $disk->{size} = $size if defined($size);
7777
7778     return $disk;
7779 }
7780
7781 sub get_running_qemu_version {
7782     my ($vmid) = @_;
7783     my $res = mon_cmd($vmid, "query-version");
7784     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7785 }
7786
7787 sub qemu_use_old_bios_files {
7788     my ($machine_type) = @_;
7789
7790     return if !$machine_type;
7791
7792     my $use_old_bios_files = undef;
7793
7794     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7795         $machine_type = $1;
7796         $use_old_bios_files = 1;
7797     } else {
7798         my $version = extract_version($machine_type, kvm_user_version());
7799         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7800         # load new efi bios files on migration. So this hack is required to allow
7801         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7802         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7803         $use_old_bios_files = !min_version($version, 2, 4);
7804     }
7805
7806     return ($use_old_bios_files, $machine_type);
7807 }
7808
7809 sub get_efivars_size {
7810     my ($conf, $efidisk) = @_;
7811
7812     my $arch = get_vm_arch($conf);
7813     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7814     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7815     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7816     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7817     return -s $ovmf_vars;
7818 }
7819
7820 sub update_efidisk_size {
7821     my ($conf) = @_;
7822
7823     return if !defined($conf->{efidisk0});
7824
7825     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7826     $disk->{size} = get_efivars_size($conf);
7827     $conf->{efidisk0} = print_drive($disk);
7828
7829     return;
7830 }
7831
7832 sub update_tpmstate_size {
7833     my ($conf) = @_;
7834
7835     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7836     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7837     $conf->{tpmstate0} = print_drive($disk);
7838 }
7839
7840 sub create_efidisk($$$$$$$) {
7841     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7842
7843     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7844     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7845
7846     my $vars_size_b = -s $ovmf_vars;
7847     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7848     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7849     PVE::Storage::activate_volumes($storecfg, [$volid]);
7850
7851     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7852     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7853
7854     return ($volid, $size/1024);
7855 }
7856
7857 sub vm_iothreads_list {
7858     my ($vmid) = @_;
7859
7860     my $res = mon_cmd($vmid, 'query-iothreads');
7861
7862     my $iothreads = {};
7863     foreach my $iothread (@$res) {
7864         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7865     }
7866
7867     return $iothreads;
7868 }
7869
7870 sub scsihw_infos {
7871     my ($conf, $drive) = @_;
7872
7873     my $maxdev = 0;
7874
7875     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7876         $maxdev = 7;
7877     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7878         $maxdev = 1;
7879     } else {
7880         $maxdev = 256;
7881     }
7882
7883     my $controller = int($drive->{index} / $maxdev);
7884     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7885         ? "virtioscsi"
7886         : "scsihw";
7887
7888     return ($maxdev, $controller, $controller_prefix);
7889 }
7890
7891 sub windows_version {
7892     my ($ostype) = @_;
7893
7894     return 0 if !$ostype;
7895
7896     my $winversion = 0;
7897
7898     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7899         $winversion = 5;
7900     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7901         $winversion = 6;
7902     } elsif ($ostype =~ m/^win(\d+)$/) {
7903         $winversion = $1;
7904     }
7905
7906     return $winversion;
7907 }
7908
7909 sub resolve_dst_disk_format {
7910         my ($storecfg, $storeid, $src_volname, $format) = @_;
7911         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7912
7913         if (!$format) {
7914             # if no target format is specified, use the source disk format as hint
7915             if ($src_volname) {
7916                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7917                 $format = qemu_img_format($scfg, $src_volname);
7918             } else {
7919                 return $defFormat;
7920             }
7921         }
7922
7923         # test if requested format is supported - else use default
7924         my $supported = grep { $_ eq $format } @$validFormats;
7925         $format = $defFormat if !$supported;
7926         return $format;
7927 }
7928
7929 # NOTE: if this logic changes, please update docs & possibly gui logic
7930 sub find_vmstate_storage {
7931     my ($conf, $storecfg) = @_;
7932
7933     # first, return storage from conf if set
7934     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7935
7936     my ($target, $shared, $local);
7937
7938     foreach_storage_used_by_vm($conf, sub {
7939         my ($sid) = @_;
7940         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7941         my $dst = $scfg->{shared} ? \$shared : \$local;
7942         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7943     });
7944
7945     # second, use shared storage where VM has at least one disk
7946     # third, use local storage where VM has at least one disk
7947     # fall back to local storage
7948     $target = $shared // $local // 'local';
7949
7950     return $target;
7951 }
7952
7953 sub generate_uuid {
7954     my ($uuid, $uuid_str);
7955     UUID::generate($uuid);
7956     UUID::unparse($uuid, $uuid_str);
7957     return $uuid_str;
7958 }
7959
7960 sub generate_smbios1_uuid {
7961     return "uuid=".generate_uuid();
7962 }
7963
7964 sub nbd_stop {
7965     my ($vmid) = @_;
7966
7967     mon_cmd($vmid, 'nbd-server-stop');
7968 }
7969
7970 sub create_reboot_request {
7971     my ($vmid) = @_;
7972     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7973         or die "failed to create reboot trigger file: $!\n";
7974     close($fh);
7975 }
7976
7977 sub clear_reboot_request {
7978     my ($vmid) = @_;
7979     my $path = "/run/qemu-server/$vmid.reboot";
7980     my $res = 0;
7981
7982     $res = unlink($path);
7983     die "could not remove reboot request for $vmid: $!"
7984         if !$res && $! != POSIX::ENOENT;
7985
7986     return $res;
7987 }
7988
7989 sub bootorder_from_legacy {
7990     my ($conf, $bootcfg) = @_;
7991
7992     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7993     my $bootindex_hash = {};
7994     my $i = 1;
7995     foreach my $o (split(//, $boot)) {
7996         $bootindex_hash->{$o} = $i*100;
7997         $i++;
7998     }
7999
8000     my $bootorder = {};
8001
8002     PVE::QemuConfig->foreach_volume($conf, sub {
8003         my ($ds, $drive) = @_;
8004
8005         if (drive_is_cdrom ($drive, 1)) {
8006             if ($bootindex_hash->{d}) {
8007                 $bootorder->{$ds} = $bootindex_hash->{d};
8008                 $bootindex_hash->{d} += 1;
8009             }
8010         } elsif ($bootindex_hash->{c}) {
8011             $bootorder->{$ds} = $bootindex_hash->{c}
8012                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8013             $bootindex_hash->{c} += 1;
8014         }
8015     });
8016
8017     if ($bootindex_hash->{n}) {
8018         for (my $i = 0; $i < $MAX_NETS; $i++) {
8019             my $netname = "net$i";
8020             next if !$conf->{$netname};
8021             $bootorder->{$netname} = $bootindex_hash->{n};
8022             $bootindex_hash->{n} += 1;
8023         }
8024     }
8025
8026     return $bootorder;
8027 }
8028
8029 # Generate default device list for 'boot: order=' property. Matches legacy
8030 # default boot order, but with explicit device names. This is important, since
8031 # the fallback for when neither 'order' nor the old format is specified relies
8032 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8033 sub get_default_bootdevices {
8034     my ($conf) = @_;
8035
8036     my @ret = ();
8037
8038     # harddisk
8039     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8040     push @ret, $first if $first;
8041
8042     # cdrom
8043     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8044     push @ret, $first if $first;
8045
8046     # network
8047     for (my $i = 0; $i < $MAX_NETS; $i++) {
8048         my $netname = "net$i";
8049         next if !$conf->{$netname};
8050         push @ret, $netname;
8051         last;
8052     }
8053
8054     return \@ret;
8055 }
8056
8057 sub device_bootorder {
8058     my ($conf) = @_;
8059
8060     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8061
8062     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8063
8064     my $bootorder = {};
8065     if (!defined($boot) || $boot->{legacy}) {
8066         $bootorder = bootorder_from_legacy($conf, $boot);
8067     } elsif ($boot->{order}) {
8068         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8069         for my $dev (PVE::Tools::split_list($boot->{order})) {
8070             $bootorder->{$dev} = $i++;
8071         }
8072     }
8073
8074     return $bootorder;
8075 }
8076
8077 sub register_qmeventd_handle {
8078     my ($vmid) = @_;
8079
8080     my $fh;
8081     my $peer = "/var/run/qmeventd.sock";
8082     my $count = 0;
8083
8084     for (;;) {
8085         $count++;
8086         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8087         last if $fh;
8088         if ($! != EINTR && $! != EAGAIN) {
8089             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8090         }
8091         if ($count > 4) {
8092             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8093               . "after $count retries\n";
8094         }
8095         usleep(25000);
8096     }
8097
8098     # send handshake to mark VM as backing up
8099     print $fh to_json({vzdump => {vmid => "$vmid"}});
8100
8101     # return handle to be closed later when inhibit is no longer required
8102     return $fh;
8103 }
8104
8105 # bash completion helper
8106
8107 sub complete_backup_archives {
8108     my ($cmdname, $pname, $cvalue) = @_;
8109
8110     my $cfg = PVE::Storage::config();
8111
8112     my $storeid;
8113
8114     if ($cvalue =~ m/^([^:]+):/) {
8115         $storeid = $1;
8116     }
8117
8118     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8119
8120     my $res = [];
8121     foreach my $id (keys %$data) {
8122         foreach my $item (@{$data->{$id}}) {
8123             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8124             push @$res, $item->{volid} if defined($item->{volid});
8125         }
8126     }
8127
8128     return $res;
8129 }
8130
8131 my $complete_vmid_full = sub {
8132     my ($running) = @_;
8133
8134     my $idlist = vmstatus();
8135
8136     my $res = [];
8137
8138     foreach my $id (keys %$idlist) {
8139         my $d = $idlist->{$id};
8140         if (defined($running)) {
8141             next if $d->{template};
8142             next if $running && $d->{status} ne 'running';
8143             next if !$running && $d->{status} eq 'running';
8144         }
8145         push @$res, $id;
8146
8147     }
8148     return $res;
8149 };
8150
8151 sub complete_vmid {
8152     return &$complete_vmid_full();
8153 }
8154
8155 sub complete_vmid_stopped {
8156     return &$complete_vmid_full(0);
8157 }
8158
8159 sub complete_vmid_running {
8160     return &$complete_vmid_full(1);
8161 }
8162
8163 sub complete_storage {
8164
8165     my $cfg = PVE::Storage::config();
8166     my $ids = $cfg->{ids};
8167
8168     my $res = [];
8169     foreach my $sid (keys %$ids) {
8170         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8171         next if !$ids->{$sid}->{content}->{images};
8172         push @$res, $sid;
8173     }
8174
8175     return $res;
8176 }
8177
8178 sub complete_migration_storage {
8179     my ($cmd, $param, $current_value, $all_args) = @_;
8180
8181     my $targetnode = @$all_args[1];
8182
8183     my $cfg = PVE::Storage::config();
8184     my $ids = $cfg->{ids};
8185
8186     my $res = [];
8187     foreach my $sid (keys %$ids) {
8188         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8189         next if !$ids->{$sid}->{content}->{images};
8190         push @$res, $sid;
8191     }
8192
8193     return $res;
8194 }
8195
8196 sub vm_is_paused {
8197     my ($vmid) = @_;
8198     my $qmpstatus = eval {
8199         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8200         mon_cmd($vmid, "query-status");
8201     };
8202     warn "$@\n" if $@;
8203     return $qmpstatus && $qmpstatus->{status} eq "paused";
8204 }
8205
8206 sub check_volume_storage_type {
8207     my ($storecfg, $vol) = @_;
8208
8209     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8210     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8211     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8212
8213     die "storage '$storeid' does not support content-type '$vtype'\n"
8214         if !$scfg->{content}->{$vtype};
8215
8216     return 1;
8217 }
8218
8219 1;