PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file(
 106     '/qemu-server/',
 107     \&parse_vm_config,
 108     \&write_vm_config
 109 );
 110
 111 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 112     description => "Some command save/restore state from this location.",
 113     type => 'string',
 114     maxLength => 128,
 115     optional => 1,
 116 });
 117
 118 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 119         description => "Specifies the QEMU machine type.",
 120         type => 'string',
 121         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 122         maxLength => 40,
 123         optional => 1,
 124 });
 125
 126 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 243             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 244             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 245             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 246             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 247             ." a hardware RNG from the host.",
 248     },
 249     max_bytes => {
 250         type => 'integer',
 251         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 252             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 253             ." `0` to disable limiting (potentially dangerous!).",
 254         optional => 1,
 255
 256         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 257         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 258         # reading from /dev/urandom
 259         default => 1024,
 260     },
 261     period => {
 262         type => 'integer',
 263         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 264             ." the guest to retrieve another 'max_bytes' of entropy.",
 265         optional => 1,
 266         default => 1000,
 267     },
 268 };
 269
 270 my $meta_info_fmt = {
 271     'ctime' => {
 272         type => 'integer',
 273         description => "The guest creation timestamp as UNIX epoch time",
 274         minimum => 0,
 275         optional => 1,
 276     },
 277     'creation-qemu' => {
 278         type => 'string',
 279         description => "The QEMU (machine) version from the time this VM was created.",
 280         pattern => '\d+(\.\d+)+',
 281         optional => 1,
 282     },
 283 };
 284
 285 my $confdesc = {
 286     onboot => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Specifies whether a VM will be started during system bootup.",
 290         default => 0,
 291     },
 292     autostart => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Automatic restart after crash (currently ignored).",
 296         default => 0,
 297     },
 298     hotplug => {
 299         optional => 1,
 300         type => 'string', format => 'pve-hotplug-features',
 301         description => "Selectively enable hotplug features. This is a comma separated list of"
 302             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 303             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 304             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 305             ." windows > 7.",
 306         default => 'network,disk,usb',
 307     },
 308     reboot => {
 309         optional => 1,
 310         type => 'boolean',
 311         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 312         default => 1,
 313     },
 314     lock => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Lock/unlock the VM.",
 318         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 319     },
 320     cpulimit => {
 321         optional => 1,
 322         type => 'number',
 323         description => "Limit of CPU usage.",
 324         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 325             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 326         minimum => 0,
 327         maximum => 128,
 328         default => 0,
 329     },
 330     cpuunits => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 334         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 335             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 336             ." weights of all the other running VMs.",
 337         minimum => 1,
 338         maximum => 262144,
 339         default => 'cgroup v1: 1024, cgroup v2: 100',
 340     },
 341     memory => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 345             ." you use the balloon device.",
 346         minimum => 16,
 347         default => 512,
 348     },
 349     balloon => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 353         minimum => 0,
 354     },
 355     shares => {
 356         optional => 1,
 357         type => 'integer',
 358         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 359             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 360             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 361         minimum => 0,
 362         maximum => 50000,
 363         default => 1000,
 364     },
 365     keyboard => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Keyboard layout for VNC server. This option is generally not required and"
 369         ." is often better handled from within the guest OS.",
 370         enum => PVE::Tools::kvmkeymaplist(),
 371         default => undef,
 372     },
 373     name => {
 374         optional => 1,
 375         type => 'string', format => 'dns-name',
 376         description => "Set a name for the VM. Only used on the configuration web interface.",
 377     },
 378     scsihw => {
 379         optional => 1,
 380         type => 'string',
 381         description => "SCSI controller model",
 382         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 383         default => 'lsi',
 384     },
 385     description => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Description for the VM. Shown in the web-interface VM's summary."
 389             ." This is saved as comment inside the configuration file.",
 390         maxLength => 1024 * 8,
 391     },
 392     ostype => {
 393         optional => 1,
 394         type => 'string',
 395         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 396         description => "Specify guest operating system.",
 397         verbose_description => <<EODESC,
 398 Specify guest operating system. This is used to enable special
 399 optimization/features for specific operating systems:
 400
 401 [horizontal]
 402 other;; unspecified OS
 403 wxp;; Microsoft Windows XP
 404 w2k;; Microsoft Windows 2000
 405 w2k3;; Microsoft Windows 2003
 406 w2k8;; Microsoft Windows 2008
 407 wvista;; Microsoft Windows Vista
 408 win7;; Microsoft Windows 7
 409 win8;; Microsoft Windows 8/2012/2012r2
 410 win10;; Microsoft Windows 10/2016/2019
 411 win11;; Microsoft Windows 11/2022
 412 l24;; Linux 2.4 Kernel
 413 l26;; Linux 2.6 - 5.X Kernel
 414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 415 EODESC
 416     },
 417     boot => {
 418         optional => 1,
 419         type => 'string', format => 'pve-qm-boot',
 420         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 421             ." key or 'legacy=' is deprecated.",
 422     },
 423     bootdisk => {
 424         optional => 1,
 425         type => 'string', format => 'pve-qm-bootdisk',
 426         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 427         pattern => '(ide|sata|scsi|virtio)\d+',
 428     },
 429     smp => {
 430         optional => 1,
 431         type => 'integer',
 432         description => "The number of CPUs. Please use option -sockets instead.",
 433         minimum => 1,
 434         default => 1,
 435     },
 436     sockets => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPU sockets.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     cores => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of cores per socket.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     numa => {
 451         optional => 1,
 452         type => 'boolean',
 453         description => "Enable/disable NUMA.",
 454         default => 0,
 455     },
 456     hugepages => {
 457         optional => 1,
 458         type => 'string',
 459         description => "Enable/disable hugepages memory.",
 460         enum => [qw(any 2 1024)],
 461     },
 462     keephugepages => {
 463         optional => 1,
 464         type => 'boolean',
 465         default => 0,
 466         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 467             ." after VM shutdown and can be used for subsequent starts.",
 468     },
 469     vcpus => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Number of hotplugged vcpus.",
 473         minimum => 1,
 474         default => 0,
 475     },
 476     acpi => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable ACPI.",
 480         default => 1,
 481     },
 482     agent => {
 483         optional => 1,
 484         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 485         type => 'string',
 486         format => $agent_fmt,
 487     },
 488     kvm => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable KVM hardware virtualization.",
 492         default => 1,
 493     },
 494     tdf => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Enable/disable time drift fix.",
 498         default => 0,
 499     },
 500     localtime => {
 501         optional => 1,
 502         type => 'boolean',
 503         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 504             ." the `ostype` indicates a Microsoft Windows OS.",
 505     },
 506     freeze => {
 507         optional => 1,
 508         type => 'boolean',
 509         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 510     },
 511     vga => {
 512         optional => 1,
 513         type => 'string', format => $vga_fmt,
 514         description => "Configure the VGA hardware.",
 515         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 516             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 517             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 518             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 519             ." display server. For win* OS you can select how many independent displays you want,"
 520             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 521             ." using a serial device as terminal.",
 522     },
 523     watchdog => {
 524         optional => 1,
 525         type => 'string', format => 'pve-qm-watchdog',
 526         description => "Create a virtual hardware watchdog device.",
 527         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 528             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 529             ." else the watchdog will reset the guest (or execute the respective action specified)",
 530     },
 531     startdate => {
 532         optional => 1,
 533         type => 'string',
 534         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 535         description => "Set the initial date of the real time clock. Valid format for date are:"
 536             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 537         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 538         default => 'now',
 539     },
 540     startup =>  get_standard_option('pve-startup-order'),
 541     template => {
 542         optional => 1,
 543         type => 'boolean',
 544         description => "Enable/disable Template.",
 545         default => 0,
 546     },
 547     args => {
 548         optional => 1,
 549         type => 'string',
 550         description => "Arbitrary arguments passed to kvm.",
 551         verbose_description => <<EODESCR,
 552 Arbitrary arguments passed to kvm, for example:
 553
 554 args: -no-reboot -no-hpet
 555
 556 NOTE: this option is for experts only.
 557 EODESCR
 558     },
 559     tablet => {
 560         optional => 1,
 561         type => 'boolean',
 562         default => 1,
 563         description => "Enable/disable the USB tablet device.",
 564         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 565             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 566             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 567             ." may consider disabling this to save some context switches. This is turned off by"
 568             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 569     },
 570     migrate_speed => {
 571         optional => 1,
 572         type => 'integer',
 573         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 574         minimum => 0,
 575         default => 0,
 576     },
 577     migrate_downtime => {
 578         optional => 1,
 579         type => 'number',
 580         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 581         minimum => 0,
 582         default => 0.1,
 583     },
 584     cdrom => {
 585         optional => 1,
 586         type => 'string', format => 'pve-qm-ide',
 587         typetext => '<volume>',
 588         description => "This is an alias for option -ide2",
 589     },
 590     cpu => {
 591         optional => 1,
 592         description => "Emulated CPU type.",
 593         type => 'string',
 594         format => 'pve-vm-cpu-conf',
 595     },
 596     parent => get_standard_option('pve-snapshot-name', {
 597         optional => 1,
 598         description => "Parent snapshot name. This is used internally, and should not be modified.",
 599     }),
 600     snaptime => {
 601         optional => 1,
 602         description => "Timestamp for snapshots.",
 603         type => 'integer',
 604         minimum => 0,
 605     },
 606     vmstate => {
 607         optional => 1,
 608         type => 'string', format => 'pve-volume-id',
 609         description => "Reference to a volume which stores the VM state. This is used internally"
 610             ." for snapshots.",
 611     },
 612     vmstatestorage => get_standard_option('pve-storage-id', {
 613         description => "Default storage for VM state volumes/files.",
 614         optional => 1,
 615     }),
 616     runningmachine => get_standard_option('pve-qemu-machine', {
 617         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 618             ." for snapshots.",
 619     }),
 620     runningcpu => {
 621         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 622             ." internally for snapshots.",
 623         optional => 1,
 624         type => 'string',
 625         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 626         format_description => 'QEMU -cpu parameter'
 627     },
 628     machine => get_standard_option('pve-qemu-machine'),
 629     arch => {
 630         description => "Virtual processor architecture. Defaults to the host.",
 631         optional => 1,
 632         type => 'string',
 633         enum => [qw(x86_64 aarch64)],
 634     },
 635     smbios1 => {
 636         description => "Specify SMBIOS type 1 fields.",
 637         type => 'string', format => 'pve-qm-smbios1',
 638         maxLength => 512,
 639         optional => 1,
 640     },
 641     protection => {
 642         optional => 1,
 643         type => 'boolean',
 644         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 645             ." remove disk operations.",
 646         default => 0,
 647     },
 648     bios => {
 649         optional => 1,
 650         type => 'string',
 651         enum => [ qw(seabios ovmf) ],
 652         description => "Select BIOS implementation.",
 653         default => 'seabios',
 654     },
 655     vmgenid => {
 656         type => 'string',
 657         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 658         format_description => 'UUID',
 659         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 660             ." to disable explicitly.",
 661         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 662             ." value identifier to the guest OS. This allows to notify the guest operating system"
 663             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 664             ." execution or creation from a template). The guest operating system notices the"
 665             ." change, and is then able to react as appropriate by marking its copies of"
 666             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 667             ."Note that auto-creation only works when done through API/CLI create or update methods"
 668             .", but not when manually editing the config file.",
 669         default => "1 (autogenerated)",
 670         optional => 1,
 671     },
 672     hookscript => {
 673         type => 'string',
 674         format => 'pve-volume-id',
 675         optional => 1,
 676         description => "Script that will be executed during various steps in the vms lifetime.",
 677     },
 678     ivshmem => {
 679         type => 'string',
 680         format => $ivshmem_fmt,
 681         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 682             ." the host.",
 683         optional => 1,
 684     },
 685     audio0 => {
 686         type => 'string',
 687         format => $audio_fmt,
 688         description => "Configure a audio device, useful in combination with QXL/Spice.",
 689         optional => 1
 690     },
 691     spice_enhancements => {
 692         type => 'string',
 693         format => $spice_enhancements_fmt,
 694         description => "Configure additional enhancements for SPICE.",
 695         optional => 1
 696     },
 697     tags => {
 698         type => 'string', format => 'pve-tag-list',
 699         description => 'Tags of the VM. This is only meta information.',
 700         optional => 1,
 701     },
 702     rng0 => {
 703         type => 'string',
 704         format => $rng_fmt,
 705         description => "Configure a VirtIO-based Random Number Generator.",
 706         optional => 1,
 707     },
 708     meta => {
 709         type => 'string',
 710         format => $meta_info_fmt,
 711         description => "Some (read-only) meta-information about this guest.",
 712         optional => 1,
 713     },
 714     affinity => {
 715         type => 'string', format => 'pve-cpuset',
 716         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 717         optional => 1,
 718     },
 719 };
 720
 721 my $cicustom_fmt = {
 722     meta => {
 723         type => 'string',
 724         optional => 1,
 725         description => 'Specify a custom file containing all meta data passed to the VM via"
 726             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 727         format => 'pve-volume-id',
 728         format_description => 'volume',
 729     },
 730     network => {
 731         type => 'string',
 732         optional => 1,
 733         description => 'Specify a custom file containing all network data passed to the VM via'
 734             .' cloud-init.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     user => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'Specify a custom file containing all user data passed to the VM via'
 742             .' cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     vendor => {
 747     type => 'string',
 748     optional => 1,
 749     description => 'Specify a custom file containing all vendor data passed to the VM via'
 750      .' cloud-init.',
 751     format => 'pve-volume-id',
 752     format_description => 'volume',
 753     },
 754 };
 755 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 756
 757 my $confdesc_cloudinit = {
 758     citype => {
 759         optional => 1,
 760         type => 'string',
 761         description => 'Specifies the cloud-init configuration format. The default depends on the'
 762             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 763             .' and `configdrive2` for windows.',
 764         enum => ['configdrive2', 'nocloud', 'opennebula'],
 765     },
 766     ciuser => {
 767         optional => 1,
 768         type => 'string',
 769         description => "cloud-init: User name to change ssh keys and password for instead of the"
 770             ." image's configured default user.",
 771     },
 772     cipassword => {
 773         optional => 1,
 774         type => 'string',
 775         description => 'cloud-init: Password to assign the user. Using this is generally not'
 776             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 777             .' support hashed passwords.',
 778     },
 779     cicustom => {
 780         optional => 1,
 781         type => 'string',
 782         description => 'cloud-init: Specify custom files to replace the automatically generated'
 783             .' ones at start.',
 784         format => 'pve-qm-cicustom',
 785     },
 786     searchdomain => {
 787         optional => 1,
 788         type => 'string',
 789         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 790             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 791             .' are set.',
 792     },
 793     nameserver => {
 794         optional => 1,
 795         type => 'string', format => 'address-list',
 796         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 797             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 798             .' are set.',
 799     },
 800     sshkeys => {
 801         optional => 1,
 802         type => 'string',
 803         format => 'urlencoded',
 804         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 805     },
 806 };
 807
 808 # what about other qemu settings ?
 809 #cpu => 'string',
 810 #machine => 'string',
 811 #fda => 'file',
 812 #fdb => 'file',
 813 #mtdblock => 'file',
 814 #sd => 'file',
 815 #pflash => 'file',
 816 #snapshot => 'bool',
 817 #bootp => 'file',
 818 ##tftp => 'dir',
 819 ##smb => 'dir',
 820 #kernel => 'file',
 821 #append => 'string',
 822 #initrd => 'file',
 823 ##soundhw => 'string',
 824
 825 while (my ($k, $v) = each %$confdesc) {
 826     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 827 }
 828
 829 my $MAX_USB_DEVICES = 14;
 830 my $MAX_NETS = 32;
 831 my $MAX_SERIAL_PORTS = 4;
 832 my $MAX_PARALLEL_PORTS = 3;
 833 my $MAX_NUMA = 8;
 834
 835 my $numa_fmt = {
 836     cpus => {
 837         type => "string",
 838         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 839         description => "CPUs accessing this NUMA node.",
 840         format_description => "id[-id];...",
 841     },
 842     memory => {
 843         type => "number",
 844         description => "Amount of memory this NUMA node provides.",
 845         optional => 1,
 846     },
 847     hostnodes => {
 848         type => "string",
 849         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 850         description => "Host NUMA nodes to use.",
 851         format_description => "id[-id];...",
 852         optional => 1,
 853     },
 854     policy => {
 855         type => 'string',
 856         enum => [qw(preferred bind interleave)],
 857         description => "NUMA allocation policy.",
 858         optional => 1,
 859     },
 860 };
 861 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 862 my $numadesc = {
 863     optional => 1,
 864     type => 'string', format => $numa_fmt,
 865     description => "NUMA topology.",
 866 };
 867 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 868
 869 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 870     $confdesc->{"numa$i"} = $numadesc;
 871 }
 872
 873 my $nic_model_list = [
 874     'e1000',
 875     'e1000-82540em',
 876     'e1000-82544gc',
 877     'e1000-82545em',
 878     'e1000e',
 879     'i82551',
 880     'i82557b',
 881     'i82559er',
 882     'ne2k_isa',
 883     'ne2k_pci',
 884     'pcnet',
 885     'rtl8139',
 886     'virtio',
 887     'vmxnet3',
 888 ];
 889 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 890
 891 my $net_fmt_bridge_descr = <<__EOD__;
 892 Bridge to attach the network device to. The Proxmox VE standard bridge
 893 is called 'vmbr0'.
 894
 895 If you do not specify a bridge, we create a kvm user (NATed) network
 896 device, which provides DHCP and DNS services. The following addresses
 897 are used:
 898
 899  10.0.2.2   Gateway
 900  10.0.2.3   DNS Server
 901  10.0.2.4   SMB Server
 902
 903 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 904 __EOD__
 905
 906 my $net_fmt = {
 907     macaddr  => get_standard_option('mac-addr', {
 908         description => "MAC address. That address must be unique withing your network. This is"
 909             ." automatically generated if not specified.",
 910     }),
 911     model => {
 912         type => 'string',
 913         description => "Network Card Model. The 'virtio' model provides the best performance with"
 914             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 915             ." best to use 'e1000'.",
 916         enum => $nic_model_list,
 917         default_key => 1,
 918     },
 919     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 920     bridge => get_standard_option('pve-bridge-id', {
 921         description => $net_fmt_bridge_descr,
 922         optional => 1,
 923     }),
 924     queues => {
 925         type => 'integer',
 926         minimum => 0, maximum => 64,
 927         description => 'Number of packet queues to be used on the device.',
 928         optional => 1,
 929     },
 930     rate => {
 931         type => 'number',
 932         minimum => 0,
 933         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 934         optional => 1,
 935     },
 936     tag => {
 937         type => 'integer',
 938         minimum => 1, maximum => 4094,
 939         description => 'VLAN tag to apply to packets on this interface.',
 940         optional => 1,
 941     },
 942     trunks => {
 943         type => 'string',
 944         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 945         description => 'VLAN trunks to pass through this interface.',
 946         format_description => 'vlanid[;vlanid...]',
 947         optional => 1,
 948     },
 949     firewall => {
 950         type => 'boolean',
 951         description => 'Whether this interface should be protected by the firewall.',
 952         optional => 1,
 953     },
 954     link_down => {
 955         type => 'boolean',
 956         description => 'Whether this interface should be disconnected (like pulling the plug).',
 957         optional => 1,
 958     },
 959     mtu => {
 960         type => 'integer',
 961         minimum => 1, maximum => 65520,
 962         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 963         optional => 1,
 964     },
 965 };
 966
 967 my $netdesc = {
 968     optional => 1,
 969     type => 'string', format => $net_fmt,
 970     description => "Specify network devices.",
 971 };
 972
 973 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 974
 975 my $ipconfig_fmt = {
 976     ip => {
 977         type => 'string',
 978         format => 'pve-ipv4-config',
 979         format_description => 'IPv4Format/CIDR',
 980         description => 'IPv4 address in CIDR format.',
 981         optional => 1,
 982         default => 'dhcp',
 983     },
 984     gw => {
 985         type => 'string',
 986         format => 'ipv4',
 987         format_description => 'GatewayIPv4',
 988         description => 'Default gateway for IPv4 traffic.',
 989         optional => 1,
 990         requires => 'ip',
 991     },
 992     ip6 => {
 993         type => 'string',
 994         format => 'pve-ipv6-config',
 995         format_description => 'IPv6Format/CIDR',
 996         description => 'IPv6 address in CIDR format.',
 997         optional => 1,
 998         default => 'dhcp',
 999     },
1000     gw6 => {
1001         type => 'string',
1002         format => 'ipv6',
1003         format_description => 'GatewayIPv6',
1004         description => 'Default gateway for IPv6 traffic.',
1005         optional => 1,
1006         requires => 'ip6',
1007     },
1008 };
1009 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1010 my $ipconfigdesc = {
1011     optional => 1,
1012     type => 'string', format => 'pve-qm-ipconfig',
1013     description => <<'EODESCR',
1014 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1015
1016 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1017
1018 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1019 gateway should be provided.
1020 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1021 cloud-init 19.4 or newer.
1022
1023 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1024 dhcp on IPv4.
1025 EODESCR
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1028
1029 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1030     $confdesc->{"net$i"} = $netdesc;
1031     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1032 }
1033
1034 foreach my $key (keys %$confdesc_cloudinit) {
1035     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1036 }
1037
1038 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1039 sub pve_verify_cpuset {
1040     my ($set_text, $noerr) = @_;
1041
1042     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1043
1044     if ($@) {
1045         return if $noerr;
1046         die "unable to parse cpuset option\n";
1047     }
1048
1049     return PVE::CpuSet->new($members)->short_string();
1050 }
1051
1052 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1053 sub verify_volume_id_or_qm_path {
1054     my ($volid, $noerr) = @_;
1055
1056     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1057
1058     return verify_volume_id_or_absolute_path($volid, $noerr);
1059 }
1060
1061 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1062 sub verify_volume_id_or_absolute_path {
1063     my ($volid, $noerr) = @_;
1064
1065     return $volid if $volid =~ m|^/|;
1066
1067     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1068     if ($@) {
1069         return if $noerr;
1070         die $@;
1071     }
1072     return $volid;
1073 }
1074
1075 my $usb_fmt = {
1076     host => {
1077         default_key => 1,
1078         type => 'string', format => 'pve-qm-usb-device',
1079         format_description => 'HOSTUSBDEVICE|spice',
1080         description => <<EODESCR,
1081 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1082
1083  'bus-port(.port)*' (decimal numbers) or
1084  'vendor_id:product_id' (hexadeciaml numbers) or
1085  'spice'
1086
1087 You can use the 'lsusb -t' command to list existing usb devices.
1088
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1090 machines - use with special care.
1091
1092 The value 'spice' can be used to add a usb redirection devices for spice.
1093 EODESCR
1094     },
1095     usb3 => {
1096         optional => 1,
1097         type => 'boolean',
1098         description => "Specifies whether if given host option is a USB3 device or port."
1099             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1100             ." is irrelevant (all devices are plugged into a xhci controller).",
1101         default => 0,
1102     },
1103 };
1104
1105 my $usbdesc = {
1106     optional => 1,
1107     type => 'string', format => $usb_fmt,
1108     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1109         ." l26 or windows > 7, n can be up to 14).",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1112
1113 my $serialdesc = {
1114         optional => 1,
1115         type => 'string',
1116         pattern => '(/dev/.+|socket)',
1117         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1118         verbose_description =>  <<EODESCR,
1119 Create a serial device inside the VM (n is 0 to 3), and pass through a
1120 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1121 host side (use 'qm terminal' to open a terminal connection).
1122
1123 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1124 use with special care.
1125
1126 CAUTION: Experimental! User reported problems with this option.
1127 EODESCR
1128 };
1129
1130 my $paralleldesc= {
1131         optional => 1,
1132         type => 'string',
1133         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1134         description =>  "Map host parallel devices (n is 0 to 2).",
1135         verbose_description =>  <<EODESCR,
1136 Map host parallel devices (n is 0 to 2).
1137
1138 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1139 machines - use with special care.
1140
1141 CAUTION: Experimental! User reported problems with this option.
1142 EODESCR
1143 };
1144
1145 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1146     $confdesc->{"parallel$i"} = $paralleldesc;
1147 }
1148
1149 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1150     $confdesc->{"serial$i"} = $serialdesc;
1151 }
1152
1153 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1154     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1155 }
1156
1157 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1158     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1159 }
1160
1161 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1162     $confdesc->{"usb$i"} = $usbdesc;
1163 }
1164
1165 my $boot_fmt = {
1166     legacy => {
1167         optional => 1,
1168         default_key => 1,
1169         type => 'string',
1170         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1171                      . " Deprecated, use 'order=' instead.",
1172         pattern => '[acdn]{1,4}',
1173         format_description => "[acdn]{1,4}",
1174
1175         # note: this is also the fallback if boot: is not given at all
1176         default => 'cdn',
1177     },
1178     order => {
1179         optional => 1,
1180         type => 'string',
1181         format => 'pve-qm-bootdev-list',
1182         format_description => "device[;device...]",
1183         description => <<EODESC,
1184 The guest will attempt to boot from devices in the order they appear here.
1185
1186 Disks, optical drives and passed-through storage USB devices will be directly
1187 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1188 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1189
1190 Note that only devices in this list will be marked as bootable and thus loaded
1191 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1192 (e.g. software-raid), you need to specify all of them here.
1193
1194 Overrides the deprecated 'legacy=[acdn]*' value when given.
1195 EODESC
1196     },
1197 };
1198 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1199
1200 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1201 sub verify_bootdev {
1202     my ($dev, $noerr) = @_;
1203
1204     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1205     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1206
1207     my $check = sub {
1208         my ($base) = @_;
1209         return 0 if $dev !~ m/^$base\d+$/;
1210         return 0 if !$confdesc->{$dev};
1211         return 1;
1212     };
1213
1214     return $dev if $check->("net");
1215     return $dev if $check->("usb");
1216     return $dev if $check->("hostpci");
1217
1218     return if $noerr;
1219     die "invalid boot device '$dev'\n";
1220 }
1221
1222 sub print_bootorder {
1223     my ($devs) = @_;
1224     return "" if !@$devs;
1225     my $data = { order => join(';', @$devs) };
1226     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1227 }
1228
1229 my $kvm_api_version = 0;
1230
1231 sub kvm_version {
1232     return $kvm_api_version if $kvm_api_version;
1233
1234     open my $fh, '<', '/dev/kvm' or return;
1235
1236     # 0xae00 => KVM_GET_API_VERSION
1237     $kvm_api_version = ioctl($fh, 0xae00, 0);
1238     close($fh);
1239
1240     return $kvm_api_version;
1241 }
1242
1243 my $kvm_user_version = {};
1244 my $kvm_mtime = {};
1245
1246 sub kvm_user_version {
1247     my ($binary) = @_;
1248
1249     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1250     my $st = stat($binary);
1251
1252     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1253     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1254         $cachedmtime == $st->mtime;
1255
1256     $kvm_user_version->{$binary} = 'unknown';
1257     $kvm_mtime->{$binary} = $st->mtime;
1258
1259     my $code = sub {
1260         my $line = shift;
1261         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1262             $kvm_user_version->{$binary} = $2;
1263         }
1264     };
1265
1266     eval { run_command([$binary, '--version'], outfunc => $code); };
1267     warn $@ if $@;
1268
1269     return $kvm_user_version->{$binary};
1270
1271 }
1272 my sub extract_version {
1273     my ($machine_type, $version) = @_;
1274     $version = kvm_user_version() if !defined($version);
1275     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1276 }
1277
1278 sub kernel_has_vhost_net {
1279     return -c '/dev/vhost-net';
1280 }
1281
1282 sub option_exists {
1283     my $key = shift;
1284     return defined($confdesc->{$key});
1285 }
1286
1287 my $cdrom_path;
1288 sub get_cdrom_path {
1289
1290     return  $cdrom_path if $cdrom_path;
1291
1292     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1293     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1294     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1295 }
1296
1297 sub get_iso_path {
1298     my ($storecfg, $vmid, $cdrom) = @_;
1299
1300     if ($cdrom eq 'cdrom') {
1301         return get_cdrom_path();
1302     } elsif ($cdrom eq 'none') {
1303         return '';
1304     } elsif ($cdrom =~ m|^/|) {
1305         return $cdrom;
1306     } else {
1307         return PVE::Storage::path($storecfg, $cdrom);
1308     }
1309 }
1310
1311 # try to convert old style file names to volume IDs
1312 sub filename_to_volume_id {
1313     my ($vmid, $file, $media) = @_;
1314
1315      if (!($file eq 'none' || $file eq 'cdrom' ||
1316           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1317
1318         return if $file =~ m|/|;
1319
1320         if ($media && $media eq 'cdrom') {
1321             $file = "local:iso/$file";
1322         } else {
1323             $file = "local:$vmid/$file";
1324         }
1325     }
1326
1327     return $file;
1328 }
1329
1330 sub verify_media_type {
1331     my ($opt, $vtype, $media) = @_;
1332
1333     return if !$media;
1334
1335     my $etype;
1336     if ($media eq 'disk') {
1337         $etype = 'images';
1338     } elsif ($media eq 'cdrom') {
1339         $etype = 'iso';
1340     } else {
1341         die "internal error";
1342     }
1343
1344     return if ($vtype eq $etype);
1345
1346     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1347 }
1348
1349 sub cleanup_drive_path {
1350     my ($opt, $storecfg, $drive) = @_;
1351
1352     # try to convert filesystem paths to volume IDs
1353
1354     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1355         ($drive->{file} !~ m|^/dev/.+|) &&
1356         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1357         ($drive->{file} !~ m/^\d+$/)) {
1358         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1359         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1360             if !$vtype;
1361         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1362         verify_media_type($opt, $vtype, $drive->{media});
1363         $drive->{file} = $volid;
1364     }
1365
1366     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1367 }
1368
1369 sub parse_hotplug_features {
1370     my ($data) = @_;
1371
1372     my $res = {};
1373
1374     return $res if $data eq '0';
1375
1376     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1377
1378     foreach my $feature (PVE::Tools::split_list($data)) {
1379         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1380             $res->{$1} = 1;
1381         } else {
1382             die "invalid hotplug feature '$feature'\n";
1383         }
1384     }
1385     return $res;
1386 }
1387
1388 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1389 sub pve_verify_hotplug_features {
1390     my ($value, $noerr) = @_;
1391
1392     return $value if parse_hotplug_features($value);
1393
1394     return if $noerr;
1395
1396     die "unable to parse hotplug option\n";
1397 }
1398
1399 sub scsi_inquiry {
1400     my($fh, $noerr) = @_;
1401
1402     my $SG_IO = 0x2285;
1403     my $SG_GET_VERSION_NUM = 0x2282;
1404
1405     my $versionbuf = "\x00" x 8;
1406     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1407     if (!$ret) {
1408         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1409         return;
1410     }
1411     my $version = unpack("I", $versionbuf);
1412     if ($version < 30000) {
1413         die "scsi generic interface too old\n"  if !$noerr;
1414         return;
1415     }
1416
1417     my $buf = "\x00" x 36;
1418     my $sensebuf = "\x00" x 8;
1419     my $cmd = pack("C x3 C x1", 0x12, 36);
1420
1421     # see /usr/include/scsi/sg.h
1422     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1423
1424     my $packet = pack(
1425         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1426     );
1427
1428     $ret = ioctl($fh, $SG_IO, $packet);
1429     if (!$ret) {
1430         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1431         return;
1432     }
1433
1434     my @res = unpack($sg_io_hdr_t, $packet);
1435     if ($res[17] || $res[18]) {
1436         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1437         return;
1438     }
1439
1440     my $res = {};
1441     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1442
1443     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1444     $res->{type} &= 0x1F;
1445
1446     return $res;
1447 }
1448
1449 sub path_is_scsi {
1450     my ($path) = @_;
1451
1452     my $fh = IO::File->new("+<$path") || return;
1453     my $res = scsi_inquiry($fh, 1);
1454     close($fh);
1455
1456     return $res;
1457 }
1458
1459 sub print_tabletdevice_full {
1460     my ($conf, $arch) = @_;
1461
1462     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1463
1464     # we use uhci for old VMs because tablet driver was buggy in older qemu
1465     my $usbbus;
1466     if ($q35 || $arch eq 'aarch64') {
1467         $usbbus = 'ehci';
1468     } else {
1469         $usbbus = 'uhci';
1470     }
1471
1472     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1473 }
1474
1475 sub print_keyboarddevice_full {
1476     my ($conf, $arch) = @_;
1477
1478     return if $arch ne 'aarch64';
1479
1480     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1481 }
1482
1483 my sub get_drive_id {
1484     my ($drive) = @_;
1485     return "$drive->{interface}$drive->{index}";
1486 }
1487
1488 sub print_drivedevice_full {
1489     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1490
1491     my $device = '';
1492     my $maxdev = 0;
1493
1494     my $drive_id = get_drive_id($drive);
1495     if ($drive->{interface} eq 'virtio') {
1496         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1497         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1498         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1499     } elsif ($drive->{interface} eq 'scsi') {
1500
1501         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1502         my $unit = $drive->{index} % $maxdev;
1503         my $devicetype = 'hd';
1504         my $path = '';
1505         if (drive_is_cdrom($drive)) {
1506             $devicetype = 'cd';
1507         } else {
1508             if ($drive->{file} =~ m|^/|) {
1509                 $path = $drive->{file};
1510                 if (my $info = path_is_scsi($path)) {
1511                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1512                         $devicetype = 'block';
1513                     } elsif ($info->{type} == 1) { # tape
1514                         $devicetype = 'generic';
1515                     }
1516                 }
1517             } else {
1518                  $path = PVE::Storage::path($storecfg, $drive->{file});
1519             }
1520
1521             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1522             my $version = extract_version($machine_type, kvm_user_version());
1523             if ($path =~ m/^iscsi\:\/\// &&
1524                !min_version($version, 4, 1)) {
1525                 $devicetype = 'generic';
1526             }
1527         }
1528
1529         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1530             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1531         } else {
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1533                 .",lun=$drive->{index}";
1534         }
1535         $device .= ",drive=drive-$drive_id,id=$drive_id";
1536
1537         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1538             $device .= ",rotation_rate=1";
1539         }
1540         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1541
1542     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1543         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1544         my $controller = int($drive->{index} / $maxdev);
1545         my $unit = $drive->{index} % $maxdev;
1546         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1547
1548         $device = "ide-$devicetype";
1549         if ($drive->{interface} eq 'ide') {
1550             $device .= ",bus=ide.$controller,unit=$unit";
1551         } else {
1552             $device .= ",bus=ahci$controller.$unit";
1553         }
1554         $device .= ",drive=drive-$drive_id,id=$drive_id";
1555
1556         if ($devicetype eq 'hd') {
1557             if (my $model = $drive->{model}) {
1558                 $model = URI::Escape::uri_unescape($model);
1559                 $device .= ",model=$model";
1560             }
1561             if ($drive->{ssd}) {
1562                 $device .= ",rotation_rate=1";
1563             }
1564         }
1565         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1566     } elsif ($drive->{interface} eq 'usb') {
1567         die "implement me";
1568         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1569     } else {
1570         die "unsupported interface type";
1571     }
1572
1573     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1574
1575     if (my $serial = $drive->{serial}) {
1576         $serial = URI::Escape::uri_unescape($serial);
1577         $device .= ",serial=$serial";
1578     }
1579
1580
1581     return $device;
1582 }
1583
1584 sub get_initiator_name {
1585     my $initiator;
1586
1587     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1588     while (defined(my $line = <$fh>)) {
1589         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1590         $initiator = $1;
1591         last;
1592     }
1593     $fh->close();
1594
1595     return $initiator;
1596 }
1597
1598 sub print_drive_commandline_full {
1599     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1600
1601     my $path;
1602     my $volid = $drive->{file};
1603     my $format = $drive->{format};
1604     my $drive_id = get_drive_id($drive);
1605
1606     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1607     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1608
1609     if (drive_is_cdrom($drive)) {
1610         $path = get_iso_path($storecfg, $vmid, $volid);
1611         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1612     } else {
1613         if ($storeid) {
1614             $path = PVE::Storage::path($storecfg, $volid);
1615             $format //= qemu_img_format($scfg, $volname);
1616         } else {
1617             $path = $volid;
1618             $format //= "raw";
1619         }
1620    }
1621
1622    my $is_rbd = $path =~ m/^rbd:/;
1623
1624     my $opts = '';
1625     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1626     foreach my $o (@qemu_drive_options) {
1627         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1628     }
1629
1630     # snapshot only accepts on|off
1631     if (defined($drive->{snapshot})) {
1632         my $v = $drive->{snapshot} ? 'on' : 'off';
1633         $opts .= ",snapshot=$v";
1634     }
1635
1636     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1637         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1638     }
1639
1640     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1641         my ($dir, $qmpname) = @$type;
1642         if (my $v = $drive->{"mbps$dir"}) {
1643             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1644         }
1645         if (my $v = $drive->{"mbps${dir}_max"}) {
1646             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1647         }
1648         if (my $v = $drive->{"bps${dir}_max_length"}) {
1649             $opts .= ",throttling.bps$qmpname-max-length=$v";
1650         }
1651         if (my $v = $drive->{"iops${dir}"}) {
1652             $opts .= ",throttling.iops$qmpname=$v";
1653         }
1654         if (my $v = $drive->{"iops${dir}_max"}) {
1655             $opts .= ",throttling.iops$qmpname-max=$v";
1656         }
1657         if (my $v = $drive->{"iops${dir}_max_length"}) {
1658             $opts .= ",throttling.iops$qmpname-max-length=$v";
1659         }
1660     }
1661
1662     if ($pbs_name) {
1663         $format = "rbd" if $is_rbd;
1664         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1665             if !$format;
1666         $opts .= ",format=alloc-track,file.driver=$format";
1667     } elsif ($format) {
1668         $opts .= ",format=$format";
1669     }
1670
1671     my $cache_direct = 0;
1672
1673     if (my $cache = $drive->{cache}) {
1674         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1675     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1676         $opts .= ",cache=none";
1677         $cache_direct = 1;
1678     }
1679
1680     # io_uring with cache mode writeback or writethrough on krbd will hang...
1681     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1682
1683     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1684     # sometimes, just plain disable...
1685     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1686
1687     # io_uring causes problems when used with CIFS since kernel 5.15
1688     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1689     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1690
1691     if (!$drive->{aio}) {
1692         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1693             # io_uring supports all cache modes
1694             $opts .= ",aio=io_uring";
1695         } else {
1696             # aio native works only with O_DIRECT
1697             if($cache_direct) {
1698                 $opts .= ",aio=native";
1699             } else {
1700                 $opts .= ",aio=threads";
1701             }
1702         }
1703     }
1704
1705     if (!drive_is_cdrom($drive)) {
1706         my $detectzeroes;
1707         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1708             $detectzeroes = 'off';
1709         } elsif ($drive->{discard}) {
1710             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1711         } else {
1712             # This used to be our default with discard not being specified:
1713             $detectzeroes = 'on';
1714         }
1715
1716         # note: 'detect-zeroes' works per blockdev and we want it to persist
1717         # after the alloc-track is removed, so put it on 'file' directly
1718         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1719         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1720     }
1721
1722     if ($pbs_name) {
1723         $opts .= ",backing=$pbs_name";
1724         $opts .= ",auto-remove=on";
1725     }
1726
1727     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1728     my $file_param = "file";
1729     if ($pbs_name) {
1730         # non-rbd drivers require the underlying file to be a seperate block
1731         # node, so add a second .file indirection
1732         $file_param .= ".file" if !$is_rbd;
1733         $file_param .= ".filename";
1734     }
1735     my $pathinfo = $path ? "$file_param=$path," : '';
1736
1737     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1738 }
1739
1740 sub print_pbs_blockdev {
1741     my ($pbs_conf, $pbs_name) = @_;
1742     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1743     $blockdev .= ",repository=$pbs_conf->{repository}";
1744     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1745     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1746     $blockdev .= ",archive=$pbs_conf->{archive}";
1747     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1748     return $blockdev;
1749 }
1750
1751 sub print_netdevice_full {
1752     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1753
1754     my $device = $net->{model};
1755     if ($net->{model} eq 'virtio') {
1756          $device = 'virtio-net-pci';
1757      };
1758
1759     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1760     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1761     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1762         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1763         # and out of each queue plus one config interrupt and control vector queue
1764         my $vectors = $net->{queues} * 2 + 2;
1765         $tmpstr .= ",vectors=$vectors,mq=on";
1766         if (min_version($machine_version, 7, 1)) {
1767             $tmpstr .= ",packed=on";
1768         }
1769     }
1770
1771     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1772         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1773     }
1774
1775     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1776
1777     if (my $mtu = $net->{mtu}) {
1778         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1779             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1780             if ($mtu == 1) {
1781                  $mtu = $bridge_mtu;
1782             } elsif ($mtu < 576) {
1783                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1784             } elsif ($mtu > $bridge_mtu) {
1785                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1786             }
1787             $tmpstr .= ",host_mtu=$mtu";
1788         } else {
1789             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1790         }
1791     }
1792
1793     if ($use_old_bios_files) {
1794         my $romfile;
1795         if ($device eq 'virtio-net-pci') {
1796             $romfile = 'pxe-virtio.rom';
1797         } elsif ($device eq 'e1000') {
1798             $romfile = 'pxe-e1000.rom';
1799         } elsif ($device eq 'e1000e') {
1800             $romfile = 'pxe-e1000e.rom';
1801         } elsif ($device eq 'ne2k') {
1802             $romfile = 'pxe-ne2k_pci.rom';
1803         } elsif ($device eq 'pcnet') {
1804             $romfile = 'pxe-pcnet.rom';
1805         } elsif ($device eq 'rtl8139') {
1806             $romfile = 'pxe-rtl8139.rom';
1807         }
1808         $tmpstr .= ",romfile=$romfile" if $romfile;
1809     }
1810
1811     return $tmpstr;
1812 }
1813
1814 sub print_netdev_full {
1815     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1816
1817     my $i = '';
1818     if ($netid =~ m/^net(\d+)$/) {
1819         $i = int($1);
1820     }
1821
1822     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1823
1824     my $ifname = "tap${vmid}i$i";
1825
1826     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1827     die "interface name '$ifname' is too long (max 15 character)\n"
1828         if length($ifname) >= 16;
1829
1830     my $vhostparam = '';
1831     if (is_native($arch)) {
1832         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1833     }
1834
1835     my $vmname = $conf->{name} || "vm$vmid";
1836
1837     my $netdev = "";
1838     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1839
1840     if ($net->{bridge}) {
1841         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1842             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1843     } else {
1844         $netdev = "type=user,id=$netid,hostname=$vmname";
1845     }
1846
1847     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1848
1849     return $netdev;
1850 }
1851
1852 my $vga_map = {
1853     'cirrus' => 'cirrus-vga',
1854     'std' => 'VGA',
1855     'vmware' => 'vmware-svga',
1856     'virtio' => 'virtio-vga',
1857     'virtio-gl' => 'virtio-vga-gl',
1858 };
1859
1860 sub print_vga_device {
1861     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1862
1863     my $type = $vga_map->{$vga->{type}};
1864     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1865         $type = 'virtio-gpu';
1866     }
1867     my $vgamem_mb = $vga->{memory};
1868
1869     my $max_outputs = '';
1870     if ($qxlnum) {
1871         $type = $id ? 'qxl' : 'qxl-vga';
1872
1873         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1874             # set max outputs so linux can have up to 4 qxl displays with one device
1875             if (min_version($machine_version, 4, 1)) {
1876                 $max_outputs = ",max_outputs=4";
1877             }
1878         }
1879     }
1880
1881     die "no devicetype for $vga->{type}\n" if !$type;
1882
1883     my $memory = "";
1884     if ($vgamem_mb) {
1885         if ($vga->{type} =~ /^virtio/) {
1886             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1887             $memory = ",max_hostmem=$bytes";
1888         } elsif ($qxlnum) {
1889             # from https://www.spice-space.org/multiple-monitors.html
1890             $memory = ",vgamem_mb=$vga->{memory}";
1891             my $ram = $vgamem_mb * 4;
1892             my $vram = $vgamem_mb * 2;
1893             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1894         } else {
1895             $memory = ",vgamem_mb=$vga->{memory}";
1896         }
1897     } elsif ($qxlnum && $id) {
1898         $memory = ",ram_size=67108864,vram_size=33554432";
1899     }
1900
1901     my $edidoff = "";
1902     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1903         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1904     }
1905
1906     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1907     my $vgaid = "vga" . ($id // '');
1908     my $pciaddr;
1909     if ($q35 && $vgaid eq 'vga') {
1910         # the first display uses pcie.0 bus on q35 machines
1911         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1912     } else {
1913         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1914     }
1915
1916     if ($vga->{type} eq 'virtio-gl') {
1917         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1918         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1919             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1920
1921         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1922             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1923     }
1924
1925     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1926 }
1927
1928 sub parse_number_sets {
1929     my ($set) = @_;
1930     my $res = [];
1931     foreach my $part (split(/;/, $set)) {
1932         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1933             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1934             push @$res, [ $1, $2 ];
1935         } else {
1936             die "invalid range: $part\n";
1937         }
1938     }
1939     return $res;
1940 }
1941
1942 sub parse_numa {
1943     my ($data) = @_;
1944
1945     my $res = parse_property_string($numa_fmt, $data);
1946     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1947     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1948     return $res;
1949 }
1950
1951 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1952 sub parse_net {
1953     my ($data, $disable_mac_autogen) = @_;
1954
1955     my $res = eval { parse_property_string($net_fmt, $data) };
1956     if ($@) {
1957         warn $@;
1958         return;
1959     }
1960     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1961         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1962         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1963     }
1964     return $res;
1965 }
1966
1967 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1968 sub parse_ipconfig {
1969     my ($data) = @_;
1970
1971     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1972     if ($@) {
1973         warn $@;
1974         return;
1975     }
1976
1977     if ($res->{gw} && !$res->{ip}) {
1978         warn 'gateway specified without specifying an IP address';
1979         return;
1980     }
1981     if ($res->{gw6} && !$res->{ip6}) {
1982         warn 'IPv6 gateway specified without specifying an IPv6 address';
1983         return;
1984     }
1985     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1986         warn 'gateway specified together with DHCP';
1987         return;
1988     }
1989     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1990         # gw6 + auto/dhcp
1991         warn "IPv6 gateway specified together with $res->{ip6} address";
1992         return;
1993     }
1994
1995     if (!$res->{ip} && !$res->{ip6}) {
1996         return { ip => 'dhcp', ip6 => 'dhcp' };
1997     }
1998
1999     return $res;
2000 }
2001
2002 sub print_net {
2003     my $net = shift;
2004
2005     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2006 }
2007
2008 sub add_random_macs {
2009     my ($settings) = @_;
2010
2011     foreach my $opt (keys %$settings) {
2012         next if $opt !~ m/^net(\d+)$/;
2013         my $net = parse_net($settings->{$opt});
2014         next if !$net;
2015         $settings->{$opt} = print_net($net);
2016     }
2017 }
2018
2019 sub vm_is_volid_owner {
2020     my ($storecfg, $vmid, $volid) = @_;
2021
2022     if ($volid !~  m|^/|) {
2023         my ($path, $owner);
2024         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2025         if ($owner && ($owner == $vmid)) {
2026             return 1;
2027         }
2028     }
2029
2030     return;
2031 }
2032
2033 sub vmconfig_register_unused_drive {
2034     my ($storecfg, $vmid, $conf, $drive) = @_;
2035
2036     if (drive_is_cloudinit($drive)) {
2037         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2038         warn $@ if $@;
2039         delete $conf->{cloudinit};
2040     } elsif (!drive_is_cdrom($drive)) {
2041         my $volid = $drive->{file};
2042         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2043             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2044         }
2045     }
2046 }
2047
2048 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2049 my $smbios1_fmt = {
2050     uuid => {
2051         type => 'string',
2052         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2053         format_description => 'UUID',
2054         description => "Set SMBIOS1 UUID.",
2055         optional => 1,
2056     },
2057     version => {
2058         type => 'string',
2059         pattern => '[A-Za-z0-9+\/]+={0,2}',
2060         format_description => 'Base64 encoded string',
2061         description => "Set SMBIOS1 version.",
2062         optional => 1,
2063     },
2064     serial => {
2065         type => 'string',
2066         pattern => '[A-Za-z0-9+\/]+={0,2}',
2067         format_description => 'Base64 encoded string',
2068         description => "Set SMBIOS1 serial number.",
2069         optional => 1,
2070     },
2071     manufacturer => {
2072         type => 'string',
2073         pattern => '[A-Za-z0-9+\/]+={0,2}',
2074         format_description => 'Base64 encoded string',
2075         description => "Set SMBIOS1 manufacturer.",
2076         optional => 1,
2077     },
2078     product => {
2079         type => 'string',
2080         pattern => '[A-Za-z0-9+\/]+={0,2}',
2081         format_description => 'Base64 encoded string',
2082         description => "Set SMBIOS1 product ID.",
2083         optional => 1,
2084     },
2085     sku => {
2086         type => 'string',
2087         pattern => '[A-Za-z0-9+\/]+={0,2}',
2088         format_description => 'Base64 encoded string',
2089         description => "Set SMBIOS1 SKU string.",
2090         optional => 1,
2091     },
2092     family => {
2093         type => 'string',
2094         pattern => '[A-Za-z0-9+\/]+={0,2}',
2095         format_description => 'Base64 encoded string',
2096         description => "Set SMBIOS1 family string.",
2097         optional => 1,
2098     },
2099     base64 => {
2100         type => 'boolean',
2101         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2102         optional => 1,
2103     },
2104 };
2105
2106 sub parse_smbios1 {
2107     my ($data) = @_;
2108
2109     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2110     warn $@ if $@;
2111     return $res;
2112 }
2113
2114 sub print_smbios1 {
2115     my ($smbios1) = @_;
2116     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2117 }
2118
2119 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2120
2121 sub parse_watchdog {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub parse_guest_agent {
2132     my ($conf) = @_;
2133
2134     return {} if !defined($conf->{agent});
2135
2136     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2137     warn $@ if $@;
2138
2139     # if the agent is disabled ignore the other potentially set properties
2140     return {} if !$res->{enabled};
2141     return $res;
2142 }
2143
2144 sub get_qga_key {
2145     my ($conf, $key) = @_;
2146     return undef if !defined($conf->{agent});
2147
2148     my $agent = parse_guest_agent($conf);
2149     return $agent->{$key};
2150 }
2151
2152 sub parse_vga {
2153     my ($value) = @_;
2154
2155     return {} if !$value;
2156     my $res = eval { parse_property_string($vga_fmt, $value) };
2157     warn $@ if $@;
2158     return $res;
2159 }
2160
2161 sub parse_rng {
2162     my ($value) = @_;
2163
2164     return if !$value;
2165
2166     my $res = eval { parse_property_string($rng_fmt, $value) };
2167     warn $@ if $@;
2168     return $res;
2169 }
2170
2171 sub parse_meta_info {
2172     my ($value) = @_;
2173
2174     return if !$value;
2175
2176     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2177     warn $@ if $@;
2178     return $res;
2179 }
2180
2181 sub new_meta_info_string {
2182     my () = @_; # for now do not allow to override any value
2183
2184     return PVE::JSONSchema::print_property_string(
2185         {
2186             'creation-qemu' => kvm_user_version(),
2187             ctime => "". int(time()),
2188         },
2189         $meta_info_fmt
2190     );
2191 }
2192
2193 sub qemu_created_version_fixups {
2194     my ($conf, $forcemachine, $kvmver) = @_;
2195
2196     my $meta = parse_meta_info($conf->{meta}) // {};
2197     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2198
2199     # check if we need to apply some handling for VMs that always use the latest machine version but
2200     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2201     # would be required (we do not want to pin machine version for non-windows OS type)
2202     if (
2203         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2204         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2205         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2206         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2207     ) {
2208         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2209         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2210             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2211             # and thus with the predictable interface naming of systemd
2212             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2213         }
2214     }
2215     return;
2216 }
2217
2218 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2219 sub verify_usb_device {
2220     my ($value, $noerr) = @_;
2221
2222     return $value if parse_usb_device($value);
2223
2224     return if $noerr;
2225
2226     die "unable to parse usb device\n";
2227 }
2228
2229 # add JSON properties for create and set function
2230 sub json_config_properties {
2231     my ($prop, $with_disk_alloc) = @_;
2232
2233     my $skip_json_config_opts = {
2234         parent => 1,
2235         snaptime => 1,
2236         vmstate => 1,
2237         runningmachine => 1,
2238         runningcpu => 1,
2239         meta => 1,
2240     };
2241
2242     foreach my $opt (keys %$confdesc) {
2243         next if $skip_json_config_opts->{$opt};
2244
2245         if ($with_disk_alloc && is_valid_drivename($opt)) {
2246             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2247         } else {
2248             $prop->{$opt} = $confdesc->{$opt};
2249         }
2250     }
2251
2252     return $prop;
2253 }
2254
2255 # Properties that we can read from an OVF file
2256 sub json_ovf_properties {
2257     my $prop = {};
2258
2259     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2260         $prop->{$device} = {
2261             type => 'string',
2262             format => 'pve-volume-id-or-absolute-path',
2263             description => "Disk image that gets imported to $device",
2264             optional => 1,
2265         };
2266     }
2267
2268     $prop->{cores} = {
2269         type => 'integer',
2270         description => "The number of CPU cores.",
2271         optional => 1,
2272     };
2273     $prop->{memory} = {
2274         type => 'integer',
2275         description => "Amount of RAM for the VM in MB.",
2276         optional => 1,
2277     };
2278     $prop->{name} = {
2279         type => 'string',
2280         description => "Name of the VM.",
2281         optional => 1,
2282     };
2283
2284     return $prop;
2285 }
2286
2287 # return copy of $confdesc_cloudinit to generate documentation
2288 sub cloudinit_config_properties {
2289
2290     return dclone($confdesc_cloudinit);
2291 }
2292
2293 sub cloudinit_pending_properties {
2294     my $p = {
2295         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2296         name => 1,
2297     };
2298     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2299     return $p;
2300 }
2301
2302 sub check_type {
2303     my ($key, $value) = @_;
2304
2305     die "unknown setting '$key'\n" if !$confdesc->{$key};
2306
2307     my $type = $confdesc->{$key}->{type};
2308
2309     if (!defined($value)) {
2310         die "got undefined value\n";
2311     }
2312
2313     if ($value =~ m/[\n\r]/) {
2314         die "property contains a line feed\n";
2315     }
2316
2317     if ($type eq 'boolean') {
2318         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2319         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2320         die "type check ('boolean') failed - got '$value'\n";
2321     } elsif ($type eq 'integer') {
2322         return int($1) if $value =~ m/^(\d+)$/;
2323         die "type check ('integer') failed - got '$value'\n";
2324     } elsif ($type eq 'number') {
2325         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2326         die "type check ('number') failed - got '$value'\n";
2327     } elsif ($type eq 'string') {
2328         if (my $fmt = $confdesc->{$key}->{format}) {
2329             PVE::JSONSchema::check_format($fmt, $value);
2330             return $value;
2331         }
2332         $value =~ s/^\"(.*)\"$/$1/;
2333         return $value;
2334     } else {
2335         die "internal error"
2336     }
2337 }
2338
2339 sub destroy_vm {
2340     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2341
2342     my $conf = PVE::QemuConfig->load_config($vmid);
2343
2344     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2345         PVE::QemuConfig->check_lock($conf);
2346     }
2347
2348     if ($conf->{template}) {
2349         # check if any base image is still used by a linked clone
2350         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2351                 my ($ds, $drive) = @_;
2352                 return if drive_is_cdrom($drive);
2353
2354                 my $volid = $drive->{file};
2355                 return if !$volid || $volid =~ m|^/|;
2356
2357                 die "base volume '$volid' is still in use by linked cloned\n"
2358                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2359
2360         });
2361     }
2362
2363     my $volids = {};
2364     my $remove_owned_drive = sub {
2365         my ($ds, $drive) = @_;
2366         return if drive_is_cdrom($drive, 1);
2367
2368         my $volid = $drive->{file};
2369         return if !$volid || $volid =~ m|^/|;
2370         return if $volids->{$volid};
2371
2372         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2373         return if !$path || !$owner || ($owner != $vmid);
2374
2375         $volids->{$volid} = 1;
2376         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2377         warn "Could not remove disk '$volid', check manually: $@" if $@;
2378     };
2379
2380     # only remove disks owned by this VM (referenced in the config)
2381     my $include_opts = {
2382         include_unused => 1,
2383         extra_keys => ['vmstate'],
2384     };
2385     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2386
2387     for my $snap (values %{$conf->{snapshots}}) {
2388         next if !defined($snap->{vmstate});
2389         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2390         next if !defined($drive);
2391         $remove_owned_drive->('vmstate', $drive);
2392     }
2393
2394     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2395
2396     if ($purge_unreferenced) { # also remove unreferenced disk
2397         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2398         PVE::Storage::foreach_volid($vmdisks, sub {
2399             my ($volid, $sid, $volname, $d) = @_;
2400             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2401             warn $@ if $@;
2402         });
2403     }
2404
2405     if (defined $replacement_conf) {
2406         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2407     } else {
2408         PVE::QemuConfig->destroy_config($vmid);
2409     }
2410 }
2411
2412 sub parse_vm_config {
2413     my ($filename, $raw, $strict) = @_;
2414
2415     return if !defined($raw);
2416
2417     my $res = {
2418         digest => Digest::SHA::sha1_hex($raw),
2419         snapshots => {},
2420         pending => {},
2421         cloudinit => {},
2422     };
2423
2424     my $handle_error = sub {
2425         my ($msg) = @_;
2426
2427         if ($strict) {
2428             die $msg;
2429         } else {
2430             warn $msg;
2431         }
2432     };
2433
2434     $filename =~ m|/qemu-server/(\d+)\.conf$|
2435         || die "got strange filename '$filename'";
2436
2437     my $vmid = $1;
2438
2439     my $conf = $res;
2440     my $descr;
2441     my $finish_description = sub {
2442         if (defined($descr)) {
2443             $descr =~ s/\s+$//;
2444             $conf->{description} = $descr;
2445         }
2446         $descr = undef;
2447     };
2448     my $section = '';
2449
2450     my @lines = split(/\n/, $raw);
2451     foreach my $line (@lines) {
2452         next if $line =~ m/^\s*$/;
2453
2454         if ($line =~ m/^\[PENDING\]\s*$/i) {
2455             $section = 'pending';
2456             $finish_description->();
2457             $conf = $res->{$section} = {};
2458             next;
2459         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2460             $section = 'cloudinit';
2461             $finish_description->();
2462             $conf = $res->{$section} = {};
2463             next;
2464
2465         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2466             $section = $1;
2467             $finish_description->();
2468             $conf = $res->{snapshots}->{$section} = {};
2469             next;
2470         }
2471
2472         if ($line =~ m/^\#(.*)$/) {
2473             $descr = '' if !defined($descr);
2474             $descr .= PVE::Tools::decode_text($1) . "\n";
2475             next;
2476         }
2477
2478         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2479             $descr = '' if !defined($descr);
2480             $descr .= PVE::Tools::decode_text($2);
2481         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2482             $conf->{snapstate} = $1;
2483         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2484             my $key = $1;
2485             my $value = $2;
2486             $conf->{$key} = $value;
2487         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2488             my $value = $1;
2489             if ($section eq 'pending') {
2490                 $conf->{delete} = $value; # we parse this later
2491             } else {
2492                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2493             }
2494         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2495             my $key = $1;
2496             my $value = $2;
2497             if ($section eq 'cloudinit') {
2498                 # ignore validation only used for informative purpose
2499                 $conf->{$key} = $value;
2500                 next;
2501             }
2502             eval { $value = check_type($key, $value); };
2503             if ($@) {
2504                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2505             } else {
2506                 $key = 'ide2' if $key eq 'cdrom';
2507                 my $fmt = $confdesc->{$key}->{format};
2508                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2509                     my $v = parse_drive($key, $value);
2510                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2511                         $v->{file} = $volid;
2512                         $value = print_drive($v);
2513                     } else {
2514                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2515                         next;
2516                     }
2517                 }
2518
2519                 $conf->{$key} = $value;
2520             }
2521         } else {
2522             $handle_error->("vm $vmid - unable to parse config: $line\n");
2523         }
2524     }
2525
2526     $finish_description->();
2527     delete $res->{snapstate}; # just to be sure
2528
2529     return $res;
2530 }
2531
2532 sub write_vm_config {
2533     my ($filename, $conf) = @_;
2534
2535     delete $conf->{snapstate}; # just to be sure
2536
2537     if ($conf->{cdrom}) {
2538         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2539         $conf->{ide2} = $conf->{cdrom};
2540         delete $conf->{cdrom};
2541     }
2542
2543     # we do not use 'smp' any longer
2544     if ($conf->{sockets}) {
2545         delete $conf->{smp};
2546     } elsif ($conf->{smp}) {
2547         $conf->{sockets} = $conf->{smp};
2548         delete $conf->{cores};
2549         delete $conf->{smp};
2550     }
2551
2552     my $used_volids = {};
2553
2554     my $cleanup_config = sub {
2555         my ($cref, $pending, $snapname) = @_;
2556
2557         foreach my $key (keys %$cref) {
2558             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2559                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2560             my $value = $cref->{$key};
2561             if ($key eq 'delete') {
2562                 die "propertry 'delete' is only allowed in [PENDING]\n"
2563                     if !$pending;
2564                 # fixme: check syntax?
2565                 next;
2566             }
2567             eval { $value = check_type($key, $value); };
2568             die "unable to parse value of '$key' - $@" if $@;
2569
2570             $cref->{$key} = $value;
2571
2572             if (!$snapname && is_valid_drivename($key)) {
2573                 my $drive = parse_drive($key, $value);
2574                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2575             }
2576         }
2577     };
2578
2579     &$cleanup_config($conf);
2580
2581     &$cleanup_config($conf->{pending}, 1);
2582
2583     foreach my $snapname (keys %{$conf->{snapshots}}) {
2584         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2585         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2586     }
2587
2588     # remove 'unusedX' settings if we re-add a volume
2589     foreach my $key (keys %$conf) {
2590         my $value = $conf->{$key};
2591         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2592             delete $conf->{$key};
2593         }
2594     }
2595
2596     my $generate_raw_config = sub {
2597         my ($conf, $pending) = @_;
2598
2599         my $raw = '';
2600
2601         # add description as comment to top of file
2602         if (defined(my $descr = $conf->{description})) {
2603             if ($descr) {
2604                 foreach my $cl (split(/\n/, $descr)) {
2605                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2606                 }
2607             } else {
2608                 $raw .= "#\n" if $pending;
2609             }
2610         }
2611
2612         foreach my $key (sort keys %$conf) {
2613             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2614             $raw .= "$key: $conf->{$key}\n";
2615         }
2616         return $raw;
2617     };
2618
2619     my $raw = &$generate_raw_config($conf);
2620
2621     if (scalar(keys %{$conf->{pending}})){
2622         $raw .= "\n[PENDING]\n";
2623         $raw .= &$generate_raw_config($conf->{pending}, 1);
2624     }
2625
2626     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2627         $raw .= "\n[special:cloudinit]\n";
2628         $raw .= &$generate_raw_config($conf->{cloudinit});
2629     }
2630
2631     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2632         $raw .= "\n[$snapname]\n";
2633         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2634     }
2635
2636     return $raw;
2637 }
2638
2639 sub load_defaults {
2640
2641     my $res = {};
2642
2643     # we use static defaults from our JSON schema configuration
2644     foreach my $key (keys %$confdesc) {
2645         if (defined(my $default = $confdesc->{$key}->{default})) {
2646             $res->{$key} = $default;
2647         }
2648     }
2649
2650     return $res;
2651 }
2652
2653 sub config_list {
2654     my $vmlist = PVE::Cluster::get_vmlist();
2655     my $res = {};
2656     return $res if !$vmlist || !$vmlist->{ids};
2657     my $ids = $vmlist->{ids};
2658     my $nodename = nodename();
2659
2660     foreach my $vmid (keys %$ids) {
2661         my $d = $ids->{$vmid};
2662         next if !$d->{node} || $d->{node} ne $nodename;
2663         next if !$d->{type} || $d->{type} ne 'qemu';
2664         $res->{$vmid}->{exists} = 1;
2665     }
2666     return $res;
2667 }
2668
2669 # test if VM uses local resources (to prevent migration)
2670 sub check_local_resources {
2671     my ($conf, $noerr) = @_;
2672
2673     my @loc_res = ();
2674
2675     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2676     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2677
2678     push @loc_res, "ivshmem" if $conf->{ivshmem};
2679
2680     foreach my $k (keys %$conf) {
2681         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2682         # sockets are safe: they will recreated be on the target side post-migrate
2683         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2684         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2685     }
2686
2687     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2688
2689     return \@loc_res;
2690 }
2691
2692 # check if used storages are available on all nodes (use by migrate)
2693 sub check_storage_availability {
2694     my ($storecfg, $conf, $node) = @_;
2695
2696     PVE::QemuConfig->foreach_volume($conf, sub {
2697         my ($ds, $drive) = @_;
2698
2699         my $volid = $drive->{file};
2700         return if !$volid;
2701
2702         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2703         return if !$sid;
2704
2705         # check if storage is available on both nodes
2706         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2707         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2708
2709         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2710
2711         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2712             if !$scfg->{content}->{$vtype};
2713    });
2714 }
2715
2716 # list nodes where all VM images are available (used by has_feature API)
2717 sub shared_nodes {
2718     my ($conf, $storecfg) = @_;
2719
2720     my $nodelist = PVE::Cluster::get_nodelist();
2721     my $nodehash = { map { $_ => 1 } @$nodelist };
2722     my $nodename = nodename();
2723
2724     PVE::QemuConfig->foreach_volume($conf, sub {
2725         my ($ds, $drive) = @_;
2726
2727         my $volid = $drive->{file};
2728         return if !$volid;
2729
2730         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2731         if ($storeid) {
2732             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2733             if ($scfg->{disable}) {
2734                 $nodehash = {};
2735             } elsif (my $avail = $scfg->{nodes}) {
2736                 foreach my $node (keys %$nodehash) {
2737                     delete $nodehash->{$node} if !$avail->{$node};
2738                 }
2739             } elsif (!$scfg->{shared}) {
2740                 foreach my $node (keys %$nodehash) {
2741                     delete $nodehash->{$node} if $node ne $nodename
2742                 }
2743             }
2744         }
2745     });
2746
2747     return $nodehash
2748 }
2749
2750 sub check_local_storage_availability {
2751     my ($conf, $storecfg) = @_;
2752
2753     my $nodelist = PVE::Cluster::get_nodelist();
2754     my $nodehash = { map { $_ => {} } @$nodelist };
2755
2756     PVE::QemuConfig->foreach_volume($conf, sub {
2757         my ($ds, $drive) = @_;
2758
2759         my $volid = $drive->{file};
2760         return if !$volid;
2761
2762         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2763         if ($storeid) {
2764             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2765
2766             if ($scfg->{disable}) {
2767                 foreach my $node (keys %$nodehash) {
2768                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2769                 }
2770             } elsif (my $avail = $scfg->{nodes}) {
2771                 foreach my $node (keys %$nodehash) {
2772                     if (!$avail->{$node}) {
2773                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2774                     }
2775                 }
2776             }
2777         }
2778     });
2779
2780     foreach my $node (values %$nodehash) {
2781         if (my $unavail = $node->{unavailable_storages}) {
2782             $node->{unavailable_storages} = [ sort keys %$unavail ];
2783         }
2784     }
2785
2786     return $nodehash
2787 }
2788
2789 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2790 sub check_running {
2791     my ($vmid, $nocheck, $node) = @_;
2792
2793     # $nocheck is set when called during a migration, in which case the config
2794     # file might still or already reside on the *other* node
2795     # - because rename has already happened, and current node is source
2796     # - because rename hasn't happened yet, and current node is target
2797     # - because rename has happened, current node is target, but hasn't yet
2798     # processed it yet
2799     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2800     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2801 }
2802
2803 sub vzlist {
2804
2805     my $vzlist = config_list();
2806
2807     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2808
2809     while (defined(my $de = $fd->read)) {
2810         next if $de !~ m/^(\d+)\.pid$/;
2811         my $vmid = $1;
2812         next if !defined($vzlist->{$vmid});
2813         if (my $pid = check_running($vmid)) {
2814             $vzlist->{$vmid}->{pid} = $pid;
2815         }
2816     }
2817
2818     return $vzlist;
2819 }
2820
2821 our $vmstatus_return_properties = {
2822     vmid => get_standard_option('pve-vmid'),
2823     status => {
2824         description => "QEMU process status.",
2825         type => 'string',
2826         enum => ['stopped', 'running'],
2827     },
2828     maxmem => {
2829         description => "Maximum memory in bytes.",
2830         type => 'integer',
2831         optional => 1,
2832         renderer => 'bytes',
2833     },
2834     maxdisk => {
2835         description => "Root disk size in bytes.",
2836         type => 'integer',
2837         optional => 1,
2838         renderer => 'bytes',
2839     },
2840     name => {
2841         description => "VM name.",
2842         type => 'string',
2843         optional => 1,
2844     },
2845     qmpstatus => {
2846         description => "QEMU QMP agent status.",
2847         type => 'string',
2848         optional => 1,
2849     },
2850     pid => {
2851         description => "PID of running qemu process.",
2852         type => 'integer',
2853         optional => 1,
2854     },
2855     uptime => {
2856         description => "Uptime.",
2857         type => 'integer',
2858         optional => 1,
2859         renderer => 'duration',
2860     },
2861     cpus => {
2862         description => "Maximum usable CPUs.",
2863         type => 'number',
2864         optional => 1,
2865     },
2866     lock => {
2867         description => "The current config lock, if any.",
2868         type => 'string',
2869         optional => 1,
2870     },
2871     tags => {
2872         description  => "The current configured tags, if any",
2873         type => 'string',
2874         optional => 1,
2875     },
2876     'running-machine' => {
2877         description => "The currently running machine type (if running).",
2878         type => 'string',
2879         optional => 1,
2880     },
2881     'running-qemu' => {
2882         description => "The currently running QEMU version (if running).",
2883         type => 'string',
2884         optional => 1,
2885     },
2886 };
2887
2888 my $last_proc_pid_stat;
2889
2890 # get VM status information
2891 # This must be fast and should not block ($full == false)
2892 # We only query KVM using QMP if $full == true (this can be slow)
2893 sub vmstatus {
2894     my ($opt_vmid, $full) = @_;
2895
2896     my $res = {};
2897
2898     my $storecfg = PVE::Storage::config();
2899
2900     my $list = vzlist();
2901     my $defaults = load_defaults();
2902
2903     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2904
2905     my $cpucount = $cpuinfo->{cpus} || 1;
2906
2907     foreach my $vmid (keys %$list) {
2908         next if $opt_vmid && ($vmid ne $opt_vmid);
2909
2910         my $conf = PVE::QemuConfig->load_config($vmid);
2911
2912         my $d = { vmid => int($vmid) };
2913         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2914
2915         # fixme: better status?
2916         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2917
2918         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2919         if (defined($size)) {
2920             $d->{disk} = 0; # no info available
2921             $d->{maxdisk} = $size;
2922         } else {
2923             $d->{disk} = 0;
2924             $d->{maxdisk} = 0;
2925         }
2926
2927         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2928             * ($conf->{cores} || $defaults->{cores});
2929         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2930         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2931
2932         $d->{name} = $conf->{name} || "VM $vmid";
2933         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2934             : $defaults->{memory}*(1024*1024);
2935
2936         if ($conf->{balloon}) {
2937             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2938             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2939                 : $defaults->{shares};
2940         }
2941
2942         $d->{uptime} = 0;
2943         $d->{cpu} = 0;
2944         $d->{mem} = 0;
2945
2946         $d->{netout} = 0;
2947         $d->{netin} = 0;
2948
2949         $d->{diskread} = 0;
2950         $d->{diskwrite} = 0;
2951
2952         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2953
2954         $d->{serial} = 1 if conf_has_serial($conf);
2955         $d->{lock} = $conf->{lock} if $conf->{lock};
2956         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2957
2958         $res->{$vmid} = $d;
2959     }
2960
2961     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2962     foreach my $dev (keys %$netdev) {
2963         next if $dev !~ m/^tap([1-9]\d*)i/;
2964         my $vmid = $1;
2965         my $d = $res->{$vmid};
2966         next if !$d;
2967
2968         $d->{netout} += $netdev->{$dev}->{receive};
2969         $d->{netin} += $netdev->{$dev}->{transmit};
2970
2971         if ($full) {
2972             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2973             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2974         }
2975
2976     }
2977
2978     my $ctime = gettimeofday;
2979
2980     foreach my $vmid (keys %$list) {
2981
2982         my $d = $res->{$vmid};
2983         my $pid = $d->{pid};
2984         next if !$pid;
2985
2986         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2987         next if !$pstat; # not running
2988
2989         my $used = $pstat->{utime} + $pstat->{stime};
2990
2991         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2992
2993         if ($pstat->{vsize}) {
2994             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2995         }
2996
2997         my $old = $last_proc_pid_stat->{$pid};
2998         if (!$old) {
2999             $last_proc_pid_stat->{$pid} = {
3000                 time => $ctime,
3001                 used => $used,
3002                 cpu => 0,
3003             };
3004             next;
3005         }
3006
3007         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3008
3009         if ($dtime > 1000) {
3010             my $dutime = $used -  $old->{used};
3011
3012             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3013             $last_proc_pid_stat->{$pid} = {
3014                 time => $ctime,
3015                 used => $used,
3016                 cpu => $d->{cpu},
3017             };
3018         } else {
3019             $d->{cpu} = $old->{cpu};
3020         }
3021     }
3022
3023     return $res if !$full;
3024
3025     my $qmpclient = PVE::QMPClient->new();
3026
3027     my $ballooncb = sub {
3028         my ($vmid, $resp) = @_;
3029
3030         my $info = $resp->{'return'};
3031         return if !$info->{max_mem};
3032
3033         my $d = $res->{$vmid};
3034
3035         # use memory assigned to VM
3036         $d->{maxmem} = $info->{max_mem};
3037         $d->{balloon} = $info->{actual};
3038
3039         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3040             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3041             $d->{freemem} = $info->{free_mem};
3042         }
3043
3044         $d->{ballooninfo} = $info;
3045     };
3046
3047     my $blockstatscb = sub {
3048         my ($vmid, $resp) = @_;
3049         my $data = $resp->{'return'} || [];
3050         my $totalrdbytes = 0;
3051         my $totalwrbytes = 0;
3052
3053         for my $blockstat (@$data) {
3054             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3055             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3056
3057             $blockstat->{device} =~ s/drive-//;
3058             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3059         }
3060         $res->{$vmid}->{diskread} = $totalrdbytes;
3061         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3062     };
3063
3064     my $machinecb = sub {
3065         my ($vmid, $resp) = @_;
3066         my $data = $resp->{'return'} || [];
3067
3068         $res->{$vmid}->{'running-machine'} =
3069             PVE::QemuServer::Machine::current_from_query_machines($data);
3070     };
3071
3072     my $versioncb = sub {
3073         my ($vmid, $resp) = @_;
3074         my $data = $resp->{'return'} // {};
3075         my $version = 'unknown';
3076
3077         if (my $v = $data->{qemu}) {
3078             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3079         }
3080
3081         $res->{$vmid}->{'running-qemu'} = $version;
3082     };
3083
3084     my $statuscb = sub {
3085         my ($vmid, $resp) = @_;
3086
3087         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3088         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3089         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3090         # this fails if ballon driver is not loaded, so this must be
3091         # the last commnand (following command are aborted if this fails).
3092         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3093
3094         my $status = 'unknown';
3095         if (!defined($status = $resp->{'return'}->{status})) {
3096             warn "unable to get VM status\n";
3097             return;
3098         }
3099
3100         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3101     };
3102
3103     foreach my $vmid (keys %$list) {
3104         next if $opt_vmid && ($vmid ne $opt_vmid);
3105         next if !$res->{$vmid}->{pid}; # not running
3106         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3107     }
3108
3109     $qmpclient->queue_execute(undef, 2);
3110
3111     foreach my $vmid (keys %$list) {
3112         next if $opt_vmid && ($vmid ne $opt_vmid);
3113         next if !$res->{$vmid}->{pid}; #not running
3114
3115         # we can't use the $qmpclient since it might have already aborted on
3116         # 'query-balloon', but this might also fail for older versions...
3117         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3118         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3119     }
3120
3121     foreach my $vmid (keys %$list) {
3122         next if $opt_vmid && ($vmid ne $opt_vmid);
3123         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3124     }
3125
3126     return $res;
3127 }
3128
3129 sub conf_has_serial {
3130     my ($conf) = @_;
3131
3132     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3133         if ($conf->{"serial$i"}) {
3134             return 1;
3135         }
3136     }
3137
3138     return 0;
3139 }
3140
3141 sub conf_has_audio {
3142     my ($conf, $id) = @_;
3143
3144     $id //= 0;
3145     my $audio = $conf->{"audio$id"};
3146     return if !defined($audio);
3147
3148     my $audioproperties = parse_property_string($audio_fmt, $audio);
3149     my $audiodriver = $audioproperties->{driver} // 'spice';
3150
3151     return {
3152         dev => $audioproperties->{device},
3153         dev_id => "audiodev$id",
3154         backend => $audiodriver,
3155         backend_id => "$audiodriver-backend${id}",
3156     };
3157 }
3158
3159 sub audio_devs {
3160     my ($audio, $audiopciaddr, $machine_version) = @_;
3161
3162     my $devs = [];
3163
3164     my $id = $audio->{dev_id};
3165     my $audiodev = "";
3166     if (min_version($machine_version, 4, 2)) {
3167         $audiodev = ",audiodev=$audio->{backend_id}";
3168     }
3169
3170     if ($audio->{dev} eq 'AC97') {
3171         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3172     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3173         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3174         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3175         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3176     } else {
3177         die "unkown audio device '$audio->{dev}', implement me!";
3178     }
3179
3180     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3181
3182     return $devs;
3183 }
3184
3185 sub get_tpm_paths {
3186     my ($vmid) = @_;
3187     return {
3188         socket => "/var/run/qemu-server/$vmid.swtpm",
3189         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3190     };
3191 }
3192
3193 sub add_tpm_device {
3194     my ($vmid, $devices, $conf) = @_;
3195
3196     return if !$conf->{tpmstate0};
3197
3198     my $paths = get_tpm_paths($vmid);
3199
3200     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3201     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3202     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3203 }
3204
3205 sub start_swtpm {
3206     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3207
3208     return if !$tpmdrive;
3209
3210     my $state;
3211     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3212     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3213     if ($storeid) {
3214         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3215     } else {
3216         $state = $tpm->{file};
3217     }
3218
3219     my $paths = get_tpm_paths($vmid);
3220
3221     # during migration, we will get state from remote
3222     #
3223     if (!$migration) {
3224         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3225         my $setup_cmd = [
3226             "swtpm_setup",
3227             "--tpmstate",
3228             "file://$state",
3229             "--createek",
3230             "--create-ek-cert",
3231             "--create-platform-cert",
3232             "--lock-nvram",
3233             "--config",
3234             "/etc/swtpm_setup.conf", # do not use XDG configs
3235             "--runas",
3236             "0", # force creation as root, error if not possible
3237             "--not-overwrite", # ignore existing state, do not modify
3238         ];
3239
3240         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3241         # TPM 2.0 supports ECC crypto, use if possible
3242         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3243
3244         run_command($setup_cmd, outfunc => sub {
3245             print "swtpm_setup: $1\n";
3246         });
3247     }
3248
3249     my $emulator_cmd = [
3250         "swtpm",
3251         "socket",
3252         "--tpmstate",
3253         "backend-uri=file://$state,mode=0600",
3254         "--ctrl",
3255         "type=unixio,path=$paths->{socket},mode=0600",
3256         "--pid",
3257         "file=$paths->{pid}",
3258         "--terminate", # terminate on QEMU disconnect
3259         "--daemon",
3260     ];
3261     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3262     run_command($emulator_cmd, outfunc => sub { print $1; });
3263
3264     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3265     while (! -e $paths->{pid}) {
3266         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3267         usleep(50_000);
3268     }
3269
3270     # return untainted PID of swtpm daemon so it can be killed on error
3271     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3272     return $1;
3273 }
3274
3275 sub vga_conf_has_spice {
3276     my ($vga) = @_;
3277
3278     my $vgaconf = parse_vga($vga);
3279     my $vgatype = $vgaconf->{type};
3280     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3281
3282     return $1 || 1;
3283 }
3284
3285 sub is_native($) {
3286     my ($arch) = @_;
3287     return get_host_arch() eq $arch;
3288 }
3289
3290 sub get_vm_arch {
3291     my ($conf) = @_;
3292     return $conf->{arch} // get_host_arch();
3293 }
3294
3295 my $default_machines = {
3296     x86_64 => 'pc',
3297     aarch64 => 'virt',
3298 };
3299
3300 sub get_installed_machine_version {
3301     my ($kvmversion) = @_;
3302     $kvmversion = kvm_user_version() if !defined($kvmversion);
3303     $kvmversion =~ m/^(\d+\.\d+)/;
3304     return $1;
3305 }
3306
3307 sub windows_get_pinned_machine_version {
3308     my ($machine, $base_version, $kvmversion) = @_;
3309
3310     my $pin_version = $base_version;
3311     if (!defined($base_version) ||
3312         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3313     ) {
3314         $pin_version = get_installed_machine_version($kvmversion);
3315     }
3316     if (!$machine || $machine eq 'pc') {
3317         $machine = "pc-i440fx-$pin_version";
3318     } elsif ($machine eq 'q35') {
3319         $machine = "pc-q35-$pin_version";
3320     } elsif ($machine eq 'virt') {
3321         $machine = "virt-$pin_version";
3322     } else {
3323         warn "unknown machine type '$machine', not touching that!\n";
3324     }
3325
3326     return $machine;
3327 }
3328
3329 sub get_vm_machine {
3330     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3331
3332     my $machine = $forcemachine || $conf->{machine};
3333
3334     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3335         $kvmversion //= kvm_user_version();
3336         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3337         # layout which confuses windows quite a bit and may result in various regressions..
3338         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3339         if (windows_version($conf->{ostype})) {
3340             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3341         }
3342         $arch //= 'x86_64';
3343         $machine ||= $default_machines->{$arch};
3344         if ($add_pve_version) {
3345             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3346             $machine .= "+pve$pvever";
3347         }
3348     }
3349
3350     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3351         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3352         $machine = $1 if $is_pxe;
3353
3354         # for version-pinned machines that do not include a pve-version (e.g.
3355         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3356         $machine .= '+pve0';
3357
3358         $machine .= '.pxe' if $is_pxe;
3359     }
3360
3361     return $machine;
3362 }
3363
3364 sub get_ovmf_files($$$) {
3365     my ($arch, $efidisk, $smm) = @_;
3366
3367     my $types = $OVMF->{$arch}
3368         or die "no OVMF images known for architecture '$arch'\n";
3369
3370     my $type = 'default';
3371     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3372         $type = $smm ? "4m" : "4m-no-smm";
3373         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3374     }
3375
3376     return $types->{$type}->@*;
3377 }
3378
3379 my $Arch2Qemu = {
3380     aarch64 => '/usr/bin/qemu-system-aarch64',
3381     x86_64 => '/usr/bin/qemu-system-x86_64',
3382 };
3383 sub get_command_for_arch($) {
3384     my ($arch) = @_;
3385     return '/usr/bin/kvm' if is_native($arch);
3386
3387     my $cmd = $Arch2Qemu->{$arch}
3388         or die "don't know how to emulate architecture '$arch'\n";
3389     return $cmd;
3390 }
3391
3392 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3393 # to use in a QEMU command line (-cpu element), first array_intersect the result
3394 # of query_supported_ with query_understood_. This is necessary because:
3395 #
3396 # a) query_understood_ returns flags the host cannot use and
3397 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3398 #    flags, but CPU settings - with most of them being flags. Those settings
3399 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3400 #
3401 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3402 # expensive. If you need the value returned from this, you can get it much
3403 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3404 # $accel being 'kvm' or 'tcg'.
3405 #
3406 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3407 # changes, automatically populating pmxcfs.
3408 #
3409 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3410 # since kvm and tcg machines support different flags
3411 #
3412 sub query_supported_cpu_flags {
3413     my ($arch) = @_;
3414
3415     $arch //= get_host_arch();
3416     my $default_machine = $default_machines->{$arch};
3417
3418     my $flags = {};
3419
3420     # FIXME: Once this is merged, the code below should work for ARM as well:
3421     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3422     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3423         $arch eq "aarch64";
3424
3425     my $kvm_supported = defined(kvm_version());
3426     my $qemu_cmd = get_command_for_arch($arch);
3427     my $fakevmid = -1;
3428     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3429
3430     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3431     my $query_supported_run_qemu = sub {
3432         my ($kvm) = @_;
3433
3434         my $flags = {};
3435         my $cmd = [
3436             $qemu_cmd,
3437             '-machine', $default_machine,
3438             '-display', 'none',
3439             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3440             '-mon', 'chardev=qmp,mode=control',
3441             '-pidfile', $pidfile,
3442             '-S', '-daemonize'
3443         ];
3444
3445         if (!$kvm) {
3446             push @$cmd, '-accel', 'tcg';
3447         }
3448
3449         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3450         die "QEMU flag querying VM exited with code " . $rc if $rc;
3451
3452         eval {
3453             my $cmd_result = mon_cmd(
3454                 $fakevmid,
3455                 'query-cpu-model-expansion',
3456                 type => 'full',
3457                 model => { name => 'host' }
3458             );
3459
3460             my $props = $cmd_result->{model}->{props};
3461             foreach my $prop (keys %$props) {
3462                 next if $props->{$prop} ne '1';
3463                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3464                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3465                 # We only keep those with underscores, to match /proc/cpuinfo
3466                 $prop =~ s/\.|-/_/g;
3467                 $flags->{$prop} = 1;
3468             }
3469         };
3470         my $err = $@;
3471
3472         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3473         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3474
3475         die $err if $err;
3476
3477         return [ sort keys %$flags ];
3478     };
3479
3480     # We need to query QEMU twice, since KVM and TCG have different supported flags
3481     PVE::QemuConfig->lock_config($fakevmid, sub {
3482         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3483         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3484
3485         if ($kvm_supported) {
3486             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3487             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3488         }
3489     });
3490
3491     return $flags;
3492 }
3493
3494 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3495 my $understood_cpu_flag_dir = "/usr/share/kvm";
3496 sub query_understood_cpu_flags {
3497     my $arch = get_host_arch();
3498     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3499
3500     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3501         if ! -e $filepath;
3502
3503     my $raw = file_get_contents($filepath);
3504     $raw =~ s/^\s+|\s+$//g;
3505     my @flags = split(/\s+/, $raw);
3506
3507     return \@flags;
3508 }
3509
3510 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3511 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3512 my sub should_disable_smm {
3513     my ($conf, $vga) = @_;
3514
3515     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3516         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3517 }
3518
3519 my sub print_ovmf_drive_commandlines {
3520     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3521
3522     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3523
3524     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3525     die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3526
3527     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3528     if ($d) {
3529         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3530         my ($path, $format) = $d->@{'file', 'format'};
3531         if ($storeid) {
3532             $path = PVE::Storage::path($storecfg, $d->{file});
3533             if (!defined($format)) {
3534                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3535                 $format = qemu_img_format($scfg, $volname);
3536             }
3537         } elsif (!defined($format)) {
3538             die "efidisk format must be specified\n";
3539         }
3540         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3541         if ($path =~ m/^rbd:/) {
3542             $var_drive_str .= ',cache=writeback';
3543             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3544         }
3545         $var_drive_str .= ",format=$format,file=$path";
3546
3547         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3548         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3549     } else {
3550         log_warn("no efidisk configured! Using temporary efivars disk.");
3551         my $path = "/tmp/$vmid-ovmf.fd";
3552         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3553         $var_drive_str .= ",format=raw,file=$path";
3554         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3555     }
3556
3557     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3558 }
3559
3560 sub config_to_command {
3561     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3562         $pbs_backing) = @_;
3563
3564     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3565     my $devices = [];
3566     my $bridges = {};
3567     my $ostype = $conf->{ostype};
3568     my $winversion = windows_version($ostype);
3569     my $kvm = $conf->{kvm};
3570     my $nodename = nodename();
3571
3572     my $arch = get_vm_arch($conf);
3573     my $kvm_binary = get_command_for_arch($arch);
3574     my $kvmver = kvm_user_version($kvm_binary);
3575
3576     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3577         $kvmver //= "undefined";
3578         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3579     }
3580
3581     my $add_pve_version = min_version($kvmver, 4, 1);
3582
3583     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3584     my $machine_version = extract_version($machine_type, $kvmver);
3585     $kvm //= 1 if is_native($arch);
3586
3587     $machine_version =~ m/(\d+)\.(\d+)/;
3588     my ($machine_major, $machine_minor) = ($1, $2);
3589
3590     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3591         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3592     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3593         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3594             ." please upgrade node '$nodename'\n"
3595     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3596         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3597         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3598             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3599             ." node '$nodename'\n";
3600     }
3601
3602     # if a specific +pve version is required for a feature, use $version_guard
3603     # instead of min_version to allow machines to be run with the minimum
3604     # required version
3605     my $required_pve_version = 0;
3606     my $version_guard = sub {
3607         my ($major, $minor, $pve) = @_;
3608         return 0 if !min_version($machine_version, $major, $minor, $pve);
3609         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3610         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3611         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3612         return 1;
3613     };
3614
3615     if ($kvm && !defined kvm_version()) {
3616         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3617             ." or enable in BIOS.\n";
3618     }
3619
3620     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3621     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3622     my $use_old_bios_files = undef;
3623     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3624
3625     my $cmd = [];
3626     if ($conf->{affinity}) {
3627         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3628     }
3629
3630     push @$cmd, $kvm_binary;
3631
3632     push @$cmd, '-id', $vmid;
3633
3634     my $vmname = $conf->{name} || "vm$vmid";
3635
3636     push @$cmd, '-name', "$vmname,debug-threads=on";
3637
3638     push @$cmd, '-no-shutdown';
3639
3640     my $use_virtio = 0;
3641
3642     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3643     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3644     push @$cmd, '-mon', "chardev=qmp,mode=control";
3645
3646     if (min_version($machine_version, 2, 12)) {
3647         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3648         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3649     }
3650
3651     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3652
3653     push @$cmd, '-daemonize';
3654
3655     if ($conf->{smbios1}) {
3656         my $smbios_conf = parse_smbios1($conf->{smbios1});
3657         if ($smbios_conf->{base64}) {
3658             # Do not pass base64 flag to qemu
3659             delete $smbios_conf->{base64};
3660             my $smbios_string = "";
3661             foreach my $key (keys %$smbios_conf) {
3662                 my $value;
3663                 if ($key eq "uuid") {
3664                     $value = $smbios_conf->{uuid}
3665                 } else {
3666                     $value = decode_base64($smbios_conf->{$key});
3667                 }
3668                 # qemu accepts any binary data, only commas need escaping by double comma
3669                 $value =~ s/,/,,/g;
3670                 $smbios_string .= "," . $key . "=" . $value if $value;
3671             }
3672             push @$cmd, '-smbios', "type=1" . $smbios_string;
3673         } else {
3674             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3675         }
3676     }
3677
3678     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3679         my ($code_drive_str, $var_drive_str) =
3680             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3681         push $cmd->@*, '-drive', $code_drive_str;
3682         push $cmd->@*, '-drive', $var_drive_str;
3683     }
3684
3685     if ($q35) { # tell QEMU to load q35 config early
3686         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3687         if (min_version($machine_version, 4, 0)) {
3688             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3689         } else {
3690             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3691         }
3692     }
3693
3694     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3695         push @$cmd, $fixups->@*;
3696     }
3697
3698     if ($conf->{vmgenid}) {
3699         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3700     }
3701
3702     # add usb controllers
3703     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3704         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3705     push @$devices, @usbcontrollers if @usbcontrollers;
3706     my $vga = parse_vga($conf->{vga});
3707
3708     my $qxlnum = vga_conf_has_spice($conf->{vga});
3709     $vga->{type} = 'qxl' if $qxlnum;
3710
3711     if (!$vga->{type}) {
3712         if ($arch eq 'aarch64') {
3713             $vga->{type} = 'virtio';
3714         } elsif (min_version($machine_version, 2, 9)) {
3715             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3716         } else {
3717             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3718         }
3719     }
3720
3721     # enable absolute mouse coordinates (needed by vnc)
3722     my $tablet = $conf->{tablet};
3723     if (!defined($tablet)) {
3724         $tablet = $defaults->{tablet};
3725         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3726         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3727     }
3728
3729     if ($tablet) {
3730         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3731         my $kbd = print_keyboarddevice_full($conf, $arch);
3732         push @$devices, '-device', $kbd if defined($kbd);
3733     }
3734
3735     my $bootorder = device_bootorder($conf);
3736
3737     # host pci device passthrough
3738     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3739         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3740
3741     # usb devices
3742     my $usb_dev_features = {};
3743     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3744
3745     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3746         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3747     push @$devices, @usbdevices if @usbdevices;
3748
3749     # serial devices
3750     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3751         my $path = $conf->{"serial$i"} or next;
3752         if ($path eq 'socket') {
3753             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3754             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3755             # On aarch64, serial0 is the UART device. QEMU only allows
3756             # connecting UART devices via the '-serial' command line, as
3757             # the device has a fixed slot on the hardware...
3758             if ($arch eq 'aarch64' && $i == 0) {
3759                 push @$devices, '-serial', "chardev:serial$i";
3760             } else {
3761                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3762             }
3763         } else {
3764             die "no such serial device\n" if ! -c $path;
3765             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3766             push @$devices, '-device', "isa-serial,chardev=serial$i";
3767         }
3768     }
3769
3770     # parallel devices
3771     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3772         if (my $path = $conf->{"parallel$i"}) {
3773             die "no such parallel device\n" if ! -c $path;
3774             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3775             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3776             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3777         }
3778     }
3779
3780     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3781         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3782         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3783         push @$devices, @$audio_devs;
3784     }
3785
3786     add_tpm_device($vmid, $devices, $conf);
3787
3788     my $sockets = 1;
3789     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3790     $sockets = $conf->{sockets} if  $conf->{sockets};
3791
3792     my $cores = $conf->{cores} || 1;
3793
3794     my $maxcpus = $sockets * $cores;
3795
3796     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3797
3798     my $allowed_vcpus = $cpuinfo->{cpus};
3799
3800     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3801
3802     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3803         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3804         for (my $i = 2; $i <= $vcpus; $i++)  {
3805             my $cpustr = print_cpu_device($conf,$i);
3806             push @$cmd, '-device', $cpustr;
3807         }
3808
3809     } else {
3810
3811         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3812     }
3813     push @$cmd, '-nodefaults';
3814
3815     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3816
3817     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3818
3819     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3820
3821     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3822         push @$devices, '-device', print_vga_device(
3823             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3824
3825         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3826
3827         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3828         push @$cmd,  '-vnc', "unix:$socket,password=on";
3829     } else {
3830         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3831         push @$cmd, '-nographic';
3832     }
3833
3834     # time drift fix
3835     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3836     my $useLocaltime = $conf->{localtime};
3837
3838     if ($winversion >= 5) { # windows
3839         $useLocaltime = 1 if !defined($conf->{localtime});
3840
3841         # use time drift fix when acpi is enabled
3842         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3843             $tdf = 1 if !defined($conf->{tdf});
3844         }
3845     }
3846
3847     if ($winversion >= 6) {
3848         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3849         push @$cmd, '-no-hpet';
3850     }
3851
3852     push @$rtcFlags, 'driftfix=slew' if $tdf;
3853
3854     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3855         push @$rtcFlags, "base=$conf->{startdate}";
3856     } elsif ($useLocaltime) {
3857         push @$rtcFlags, 'base=localtime';
3858     }
3859
3860     if ($forcecpu) {
3861         push @$cmd, '-cpu', $forcecpu;
3862     } else {
3863         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3864     }
3865
3866     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3867
3868     push @$cmd, '-S' if $conf->{freeze};
3869
3870     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3871
3872     my $guest_agent = parse_guest_agent($conf);
3873
3874     if ($guest_agent->{enabled}) {
3875         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3876         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3877
3878         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3879             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3880             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3881             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3882         } elsif ($guest_agent->{type} eq 'isa') {
3883             push @$devices, '-device', "isa-serial,chardev=qga0";
3884         }
3885     }
3886
3887     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3888     if ($rng && $version_guard->(4, 1, 2)) {
3889         check_rng_source($rng->{source});
3890
3891         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3892         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3893         my $limiter_str = "";
3894         if ($max_bytes) {
3895             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3896         }
3897
3898         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3899         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3900         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3901     }
3902
3903     my $spice_port;
3904
3905     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3906         if ($qxlnum > 1) {
3907             if ($winversion){
3908                 for (my $i = 1; $i < $qxlnum; $i++){
3909                     push @$devices, '-device', print_vga_device(
3910                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3911                 }
3912             } else {
3913                 # assume other OS works like Linux
3914                 my ($ram, $vram) = ("134217728", "67108864");
3915                 if ($vga->{memory}) {
3916                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3917                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3918                 }
3919                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3920                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3921             }
3922         }
3923
3924         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3925
3926         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3927         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3928         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3929
3930         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3931         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3932         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3933
3934         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3935         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3936
3937         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3938         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3939         if ($spice_enhancement->{foldersharing}) {
3940             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3941             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3942         }
3943
3944         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3945         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3946             if $spice_enhancement->{videostreaming};
3947
3948         push @$devices, '-spice', "$spice_opts";
3949     }
3950
3951     # enable balloon by default, unless explicitly disabled
3952     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3953         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3954         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3955         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3956         push @$devices, '-device', $ballooncmd;
3957     }
3958
3959     if ($conf->{watchdog}) {
3960         my $wdopts = parse_watchdog($conf->{watchdog});
3961         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3962         my $watchdog = $wdopts->{model} || 'i6300esb';
3963         push @$devices, '-device', "$watchdog$pciaddr";
3964         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3965     }
3966
3967     my $vollist = [];
3968     my $scsicontroller = {};
3969     my $ahcicontroller = {};
3970     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3971
3972     # Add iscsi initiator name if available
3973     if (my $initiator = get_initiator_name()) {
3974         push @$devices, '-iscsi', "initiator-name=$initiator";
3975     }
3976
3977     PVE::QemuConfig->foreach_volume($conf, sub {
3978         my ($ds, $drive) = @_;
3979
3980         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3981             check_volume_storage_type($storecfg, $drive->{file});
3982             push @$vollist, $drive->{file};
3983         }
3984
3985         # ignore efidisk here, already added in bios/fw handling code above
3986         return if $drive->{interface} eq 'efidisk';
3987         # similar for TPM
3988         return if $drive->{interface} eq 'tpmstate';
3989
3990         $use_virtio = 1 if $ds =~ m/^virtio/;
3991
3992         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3993
3994         if ($drive->{interface} eq 'virtio'){
3995            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3996         }
3997
3998         if ($drive->{interface} eq 'scsi') {
3999
4000             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4001
4002             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4003                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4004
4005             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4006             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4007
4008             my $iothread = '';
4009             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4010                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4011                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4012             } elsif ($drive->{iothread}) {
4013                 log_warn(
4014                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4015                 );
4016             }
4017
4018             my $queues = '';
4019             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4020                 $queues = ",num_queues=$drive->{queues}";
4021             }
4022
4023             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4024                 if !$scsicontroller->{$controller};
4025             $scsicontroller->{$controller}=1;
4026         }
4027
4028         if ($drive->{interface} eq 'sata') {
4029             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4030             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4031             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4032                 if !$ahcicontroller->{$controller};
4033             $ahcicontroller->{$controller}=1;
4034         }
4035
4036         my $pbs_conf = $pbs_backing->{$ds};
4037         my $pbs_name = undef;
4038         if ($pbs_conf) {
4039             $pbs_name = "drive-$ds-pbs";
4040             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4041         }
4042
4043         my $drive_cmd = print_drive_commandline_full(
4044             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4045
4046         # extra protection for templates, but SATA and IDE don't support it..
4047         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4048
4049         push @$devices, '-drive',$drive_cmd;
4050         push @$devices, '-device', print_drivedevice_full(
4051             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4052     });
4053
4054     for (my $i = 0; $i < $MAX_NETS; $i++) {
4055         my $netname = "net$i";
4056
4057         next if !$conf->{$netname};
4058         my $d = parse_net($conf->{$netname});
4059         next if !$d;
4060         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4061
4062         $use_virtio = 1 if $d->{model} eq 'virtio';
4063
4064         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4065
4066         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4067         push @$devices, '-netdev', $netdevfull;
4068
4069         my $netdevicefull = print_netdevice_full(
4070             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4071
4072         push @$devices, '-device', $netdevicefull;
4073     }
4074
4075     if ($conf->{ivshmem}) {
4076         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4077
4078         my $bus;
4079         if ($q35) {
4080             $bus = print_pcie_addr("ivshmem");
4081         } else {
4082             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4083         }
4084
4085         my $ivshmem_name = $ivshmem->{name} // $vmid;
4086         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4087
4088         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4089         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4090             .",size=$ivshmem->{size}M";
4091     }
4092
4093     # pci.4 is nested in pci.1
4094     $bridges->{1} = 1 if $bridges->{4};
4095
4096     if (!$q35) { # add pci bridges
4097         if (min_version($machine_version, 2, 3)) {
4098            $bridges->{1} = 1;
4099            $bridges->{2} = 1;
4100         }
4101         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4102     }
4103
4104     for my $k (sort {$b cmp $a} keys %$bridges) {
4105         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4106
4107         my $k_name = $k;
4108         if ($k == 2 && $legacy_igd) {
4109             $k_name = "$k-igd";
4110         }
4111         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4112         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4113
4114         if ($q35) { # add after -readconfig pve-q35.cfg
4115             splice @$devices, 2, 0, '-device', $devstr;
4116         } else {
4117             unshift @$devices, '-device', $devstr if $k > 0;
4118         }
4119     }
4120
4121     if (!$kvm) {
4122         push @$machineFlags, 'accel=tcg';
4123     }
4124
4125     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4126
4127     my $machine_type_min = $machine_type;
4128     if ($add_pve_version) {
4129         $machine_type_min =~ s/\+pve\d+$//;
4130         $machine_type_min .= "+pve$required_pve_version";
4131     }
4132     push @$machineFlags, "type=${machine_type_min}";
4133
4134     push @$cmd, @$devices;
4135     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4136     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4137     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4138
4139     if (my $vmstate = $conf->{vmstate}) {
4140         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4141         push @$vollist, $vmstate;
4142         push @$cmd, '-loadstate', $statepath;
4143         print "activating and using '$vmstate' as vmstate\n";
4144     }
4145
4146     if (PVE::QemuConfig->is_template($conf)) {
4147         # needed to workaround base volumes being read-only
4148         push @$cmd, '-snapshot';
4149     }
4150
4151     # add custom args
4152     if ($conf->{args}) {
4153         my $aa = PVE::Tools::split_args($conf->{args});
4154         push @$cmd, @$aa;
4155     }
4156
4157     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4158 }
4159
4160 sub check_rng_source {
4161     my ($source) = @_;
4162
4163     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4164     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4165         if ! -e $source;
4166
4167     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4168     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4169         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4170         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4171         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4172             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4173             ." to the host.\n";
4174     }
4175 }
4176
4177 sub spice_port {
4178     my ($vmid) = @_;
4179
4180     my $res = mon_cmd($vmid, 'query-spice');
4181
4182     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4183 }
4184
4185 sub vm_devices_list {
4186     my ($vmid) = @_;
4187
4188     my $res = mon_cmd($vmid, 'query-pci');
4189     my $devices_to_check = [];
4190     my $devices = {};
4191     foreach my $pcibus (@$res) {
4192         push @$devices_to_check, @{$pcibus->{devices}},
4193     }
4194
4195     while (@$devices_to_check) {
4196         my $to_check = [];
4197         for my $d (@$devices_to_check) {
4198             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4199             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4200
4201             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4202             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4203         }
4204         $devices_to_check = $to_check;
4205     }
4206
4207     my $resblock = mon_cmd($vmid, 'query-block');
4208     foreach my $block (@$resblock) {
4209         if($block->{device} =~ m/^drive-(\S+)/){
4210                 $devices->{$1} = 1;
4211         }
4212     }
4213
4214     my $resmice = mon_cmd($vmid, 'query-mice');
4215     foreach my $mice (@$resmice) {
4216         if ($mice->{name} eq 'QEMU HID Tablet') {
4217             $devices->{tablet} = 1;
4218             last;
4219         }
4220     }
4221
4222     # for usb devices there is no query-usb
4223     # but we can iterate over the entries in
4224     # qom-list path=/machine/peripheral
4225     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4226     foreach my $per (@$resperipheral) {
4227         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4228             $devices->{$per->{name}} = 1;
4229         }
4230     }
4231
4232     return $devices;
4233 }
4234
4235 sub vm_deviceplug {
4236     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4237
4238     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4239
4240     my $devices_list = vm_devices_list($vmid);
4241     return 1 if defined($devices_list->{$deviceid});
4242
4243     # add PCI bridge if we need it for the device
4244     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4245
4246     if ($deviceid eq 'tablet') {
4247         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4248     } elsif ($deviceid eq 'keyboard') {
4249         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4250     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4251         my $id = $1;
4252         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4253         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4254     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4255         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4256     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4257         qemu_iothread_add($vmid, $deviceid, $device);
4258
4259         qemu_driveadd($storecfg, $vmid, $device);
4260         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4261
4262         qemu_deviceadd($vmid, $devicefull);
4263         eval { qemu_deviceaddverify($vmid, $deviceid); };
4264         if (my $err = $@) {
4265             eval { qemu_drivedel($vmid, $deviceid); };
4266             warn $@ if $@;
4267             die $err;
4268         }
4269     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4270         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4271         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4272         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4273
4274         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4275
4276         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4277             qemu_iothread_add($vmid, $deviceid, $device);
4278             $devicefull .= ",iothread=iothread-$deviceid";
4279         }
4280
4281         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4282             $devicefull .= ",num_queues=$device->{queues}";
4283         }
4284
4285         qemu_deviceadd($vmid, $devicefull);
4286         qemu_deviceaddverify($vmid, $deviceid);
4287     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4288         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4289         qemu_driveadd($storecfg, $vmid, $device);
4290
4291         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4292         eval { qemu_deviceadd($vmid, $devicefull); };
4293         if (my $err = $@) {
4294             eval { qemu_drivedel($vmid, $deviceid); };
4295             warn $@ if $@;
4296             die $err;
4297         }
4298     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4299         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4300
4301         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4302         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4303         my $use_old_bios_files = undef;
4304         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4305
4306         my $netdevicefull = print_netdevice_full(
4307             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4308         qemu_deviceadd($vmid, $netdevicefull);
4309         eval {
4310             qemu_deviceaddverify($vmid, $deviceid);
4311             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4312         };
4313         if (my $err = $@) {
4314             eval { qemu_netdevdel($vmid, $deviceid); };
4315             warn $@ if $@;
4316             die $err;
4317         }
4318     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4319         my $bridgeid = $2;
4320         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4321         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4322
4323         qemu_deviceadd($vmid, $devicefull);
4324         qemu_deviceaddverify($vmid, $deviceid);
4325     } else {
4326         die "can't hotplug device '$deviceid'\n";
4327     }
4328
4329     return 1;
4330 }
4331
4332 # fixme: this should raise exceptions on error!
4333 sub vm_deviceunplug {
4334     my ($vmid, $conf, $deviceid) = @_;
4335
4336     my $devices_list = vm_devices_list($vmid);
4337     return 1 if !defined($devices_list->{$deviceid});
4338
4339     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4340     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4341
4342     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4343         qemu_devicedel($vmid, $deviceid);
4344     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4345         qemu_devicedel($vmid, $deviceid);
4346         qemu_devicedelverify($vmid, $deviceid);
4347     } elsif ($deviceid =~ m/^usb\d+$/) {
4348         qemu_devicedel($vmid, $deviceid);
4349         qemu_devicedelverify($vmid, $deviceid);
4350     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4351         my $device = parse_drive($deviceid, $conf->{$deviceid});
4352
4353         qemu_devicedel($vmid, $deviceid);
4354         qemu_devicedelverify($vmid, $deviceid);
4355         qemu_drivedel($vmid, $deviceid);
4356         qemu_iothread_del($vmid, $deviceid, $device);
4357     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4358         qemu_devicedel($vmid, $deviceid);
4359         qemu_devicedelverify($vmid, $deviceid);
4360     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4361         my $device = parse_drive($deviceid, $conf->{$deviceid});
4362
4363         qemu_devicedel($vmid, $deviceid);
4364         qemu_devicedelverify($vmid, $deviceid);
4365         qemu_drivedel($vmid, $deviceid);
4366         qemu_deletescsihw($conf, $vmid, $deviceid);
4367
4368         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4369             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4370     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4371         qemu_devicedel($vmid, $deviceid);
4372         qemu_devicedelverify($vmid, $deviceid);
4373         qemu_netdevdel($vmid, $deviceid);
4374     } else {
4375         die "can't unplug device '$deviceid'\n";
4376     }
4377
4378     return 1;
4379 }
4380
4381 sub qemu_spice_usbredir_chardev_add {
4382     my ($vmid, $id) = @_;
4383
4384     mon_cmd($vmid, "chardev-add" , (
4385         id => $id,
4386         backend => {
4387             type => 'spicevmc',
4388             data => {
4389                 type => "usbredir",
4390             },
4391         },
4392     ));
4393 }
4394
4395 sub qemu_deviceadd {
4396     my ($vmid, $devicefull) = @_;
4397
4398     $devicefull = "driver=".$devicefull;
4399     my %options =  split(/[=,]/, $devicefull);
4400
4401     mon_cmd($vmid, "device_add" , %options);
4402 }
4403
4404 sub qemu_devicedel {
4405     my ($vmid, $deviceid) = @_;
4406
4407     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4408 }
4409
4410 sub qemu_iothread_add {
4411     my ($vmid, $deviceid, $device) = @_;
4412
4413     if ($device->{iothread}) {
4414         my $iothreads = vm_iothreads_list($vmid);
4415         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4416     }
4417 }
4418
4419 sub qemu_iothread_del {
4420     my ($vmid, $deviceid, $device) = @_;
4421
4422     if ($device->{iothread}) {
4423         my $iothreads = vm_iothreads_list($vmid);
4424         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4425     }
4426 }
4427
4428 sub qemu_objectadd {
4429     my ($vmid, $objectid, $qomtype) = @_;
4430
4431     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4432
4433     return 1;
4434 }
4435
4436 sub qemu_objectdel {
4437     my ($vmid, $objectid) = @_;
4438
4439     mon_cmd($vmid, "object-del", id => $objectid);
4440
4441     return 1;
4442 }
4443
4444 sub qemu_driveadd {
4445     my ($storecfg, $vmid, $device) = @_;
4446
4447     my $kvmver = get_running_qemu_version($vmid);
4448     my $io_uring = min_version($kvmver, 6, 0);
4449     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4450     $drive =~ s/\\/\\\\/g;
4451     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4452
4453     # If the command succeeds qemu prints: "OK"
4454     return 1 if $ret =~ m/OK/s;
4455
4456     die "adding drive failed: $ret\n";
4457 }
4458
4459 sub qemu_drivedel {
4460     my ($vmid, $deviceid) = @_;
4461
4462     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4463     $ret =~ s/^\s+//;
4464
4465     return 1 if $ret eq "";
4466
4467     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4468     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4469
4470     die "deleting drive $deviceid failed : $ret\n";
4471 }
4472
4473 sub qemu_deviceaddverify {
4474     my ($vmid, $deviceid) = @_;
4475
4476     for (my $i = 0; $i <= 5; $i++) {
4477          my $devices_list = vm_devices_list($vmid);
4478          return 1 if defined($devices_list->{$deviceid});
4479          sleep 1;
4480     }
4481
4482     die "error on hotplug device '$deviceid'\n";
4483 }
4484
4485
4486 sub qemu_devicedelverify {
4487     my ($vmid, $deviceid) = @_;
4488
4489     # need to verify that the device is correctly removed as device_del
4490     # is async and empty return is not reliable
4491
4492     for (my $i = 0; $i <= 5; $i++) {
4493          my $devices_list = vm_devices_list($vmid);
4494          return 1 if !defined($devices_list->{$deviceid});
4495          sleep 1;
4496     }
4497
4498     die "error on hot-unplugging device '$deviceid'\n";
4499 }
4500
4501 sub qemu_findorcreatescsihw {
4502     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4503
4504     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4505
4506     my $scsihwid="$controller_prefix$controller";
4507     my $devices_list = vm_devices_list($vmid);
4508
4509     if (!defined($devices_list->{$scsihwid})) {
4510         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4511     }
4512
4513     return 1;
4514 }
4515
4516 sub qemu_deletescsihw {
4517     my ($conf, $vmid, $opt) = @_;
4518
4519     my $device = parse_drive($opt, $conf->{$opt});
4520
4521     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4522         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4523         return 1;
4524     }
4525
4526     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4527
4528     my $devices_list = vm_devices_list($vmid);
4529     foreach my $opt (keys %{$devices_list}) {
4530         if (is_valid_drivename($opt)) {
4531             my $drive = parse_drive($opt, $conf->{$opt});
4532             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4533                 return 1;
4534             }
4535         }
4536     }
4537
4538     my $scsihwid="scsihw$controller";
4539
4540     vm_deviceunplug($vmid, $conf, $scsihwid);
4541
4542     return 1;
4543 }
4544
4545 sub qemu_add_pci_bridge {
4546     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4547
4548     my $bridges = {};
4549
4550     my $bridgeid;
4551
4552     print_pci_addr($device, $bridges, $arch, $machine_type);
4553
4554     while (my ($k, $v) = each %$bridges) {
4555         $bridgeid = $k;
4556     }
4557     return 1 if !defined($bridgeid) || $bridgeid < 1;
4558
4559     my $bridge = "pci.$bridgeid";
4560     my $devices_list = vm_devices_list($vmid);
4561
4562     if (!defined($devices_list->{$bridge})) {
4563         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4564     }
4565
4566     return 1;
4567 }
4568
4569 sub qemu_set_link_status {
4570     my ($vmid, $device, $up) = @_;
4571
4572     mon_cmd($vmid, "set_link", name => $device,
4573                up => $up ? JSON::true : JSON::false);
4574 }
4575
4576 sub qemu_netdevadd {
4577     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4578
4579     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4580     my %options =  split(/[=,]/, $netdev);
4581
4582     if (defined(my $vhost = $options{vhost})) {
4583         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4584     }
4585
4586     if (defined(my $queues = $options{queues})) {
4587         $options{queues} = $queues + 0;
4588     }
4589
4590     mon_cmd($vmid, "netdev_add",  %options);
4591     return 1;
4592 }
4593
4594 sub qemu_netdevdel {
4595     my ($vmid, $deviceid) = @_;
4596
4597     mon_cmd($vmid, "netdev_del", id => $deviceid);
4598 }
4599
4600 sub qemu_usb_hotplug {
4601     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4602
4603     return if !$device;
4604
4605     # remove the old one first
4606     vm_deviceunplug($vmid, $conf, $deviceid);
4607
4608     # check if xhci controller is necessary and available
4609     my $devicelist = vm_devices_list($vmid);
4610
4611     if (!$devicelist->{xhci}) {
4612         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4613         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4614     }
4615
4616     # print_usbdevice_full expects the parsed device
4617     my $d = parse_usb_device($device->{host});
4618     $d->{usb3} = $device->{usb3};
4619
4620     # add the new one
4621     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4622 }
4623
4624 sub qemu_cpu_hotplug {
4625     my ($vmid, $conf, $vcpus) = @_;
4626
4627     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4628
4629     my $sockets = 1;
4630     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4631     $sockets = $conf->{sockets} if  $conf->{sockets};
4632     my $cores = $conf->{cores} || 1;
4633     my $maxcpus = $sockets * $cores;
4634
4635     $vcpus = $maxcpus if !$vcpus;
4636
4637     die "you can't add more vcpus than maxcpus\n"
4638         if $vcpus > $maxcpus;
4639
4640     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4641
4642     if ($vcpus < $currentvcpus) {
4643
4644         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4645
4646             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4647                 qemu_devicedel($vmid, "cpu$i");
4648                 my $retry = 0;
4649                 my $currentrunningvcpus = undef;
4650                 while (1) {
4651                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4652                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4653                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4654                     $retry++;
4655                     sleep 1;
4656                 }
4657                 #update conf after each succesfull cpu unplug
4658                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4659                 PVE::QemuConfig->write_config($vmid, $conf);
4660             }
4661         } else {
4662             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4663         }
4664
4665         return;
4666     }
4667
4668     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4669     die "vcpus in running vm does not match its configuration\n"
4670         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4671
4672     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4673
4674         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4675             my $cpustr = print_cpu_device($conf, $i);
4676             qemu_deviceadd($vmid, $cpustr);
4677
4678             my $retry = 0;
4679             my $currentrunningvcpus = undef;
4680             while (1) {
4681                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4682                 last if scalar(@{$currentrunningvcpus}) == $i;
4683                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4684                 sleep 1;
4685                 $retry++;
4686             }
4687             #update conf after each succesfull cpu hotplug
4688             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4689             PVE::QemuConfig->write_config($vmid, $conf);
4690         }
4691     } else {
4692
4693         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4694             mon_cmd($vmid, "cpu-add", id => int($i));
4695         }
4696     }
4697 }
4698
4699 sub qemu_block_set_io_throttle {
4700     my ($vmid, $deviceid,
4701         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4702         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4703         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4704         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4705
4706     return if !check_running($vmid) ;
4707
4708     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4709         bps => int($bps),
4710         bps_rd => int($bps_rd),
4711         bps_wr => int($bps_wr),
4712         iops => int($iops),
4713         iops_rd => int($iops_rd),
4714         iops_wr => int($iops_wr),
4715         bps_max => int($bps_max),
4716         bps_rd_max => int($bps_rd_max),
4717         bps_wr_max => int($bps_wr_max),
4718         iops_max => int($iops_max),
4719         iops_rd_max => int($iops_rd_max),
4720         iops_wr_max => int($iops_wr_max),
4721         bps_max_length => int($bps_max_length),
4722         bps_rd_max_length => int($bps_rd_max_length),
4723         bps_wr_max_length => int($bps_wr_max_length),
4724         iops_max_length => int($iops_max_length),
4725         iops_rd_max_length => int($iops_rd_max_length),
4726         iops_wr_max_length => int($iops_wr_max_length),
4727     );
4728
4729 }
4730
4731 sub qemu_block_resize {
4732     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4733
4734     my $running = check_running($vmid);
4735
4736     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4737
4738     return if !$running;
4739
4740     my $padding = (1024 - $size % 1024) % 1024;
4741     $size = $size + $padding;
4742
4743     mon_cmd(
4744         $vmid,
4745         "block_resize",
4746         device => $deviceid,
4747         size => int($size),
4748         timeout => 60,
4749     );
4750 }
4751
4752 sub qemu_volume_snapshot {
4753     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4754
4755     my $running = check_running($vmid);
4756
4757     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4758         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4759     } else {
4760         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4761     }
4762 }
4763
4764 sub qemu_volume_snapshot_delete {
4765     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4766
4767     my $running = check_running($vmid);
4768
4769     if($running) {
4770
4771         $running = undef;
4772         my $conf = PVE::QemuConfig->load_config($vmid);
4773         PVE::QemuConfig->foreach_volume($conf, sub {
4774             my ($ds, $drive) = @_;
4775             $running = 1 if $drive->{file} eq $volid;
4776         });
4777     }
4778
4779     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4780         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4781     } else {
4782         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4783     }
4784 }
4785
4786 sub set_migration_caps {
4787     my ($vmid, $savevm) = @_;
4788
4789     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4790
4791     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4792     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4793
4794     my $cap_ref = [];
4795
4796     my $enabled_cap = {
4797         "auto-converge" => 1,
4798         "xbzrle" => 1,
4799         "x-rdma-pin-all" => 0,
4800         "zero-blocks" => 0,
4801         "compress" => 0,
4802         "dirty-bitmaps" => $dirty_bitmaps,
4803     };
4804
4805     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4806
4807     for my $supported_capability (@$supported_capabilities) {
4808         push @$cap_ref, {
4809             capability => $supported_capability->{capability},
4810             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4811         };
4812     }
4813
4814     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4815 }
4816
4817 sub foreach_volid {
4818     my ($conf, $func, @param) = @_;
4819
4820     my $volhash = {};
4821
4822     my $test_volid = sub {
4823         my ($key, $drive, $snapname) = @_;
4824
4825         my $volid = $drive->{file};
4826         return if !$volid;
4827
4828         $volhash->{$volid}->{cdrom} //= 1;
4829         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4830
4831         my $replicate = $drive->{replicate} // 1;
4832         $volhash->{$volid}->{replicate} //= 0;
4833         $volhash->{$volid}->{replicate} = 1 if $replicate;
4834
4835         $volhash->{$volid}->{shared} //= 0;
4836         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4837
4838         $volhash->{$volid}->{referenced_in_config} //= 0;
4839         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4840
4841         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4842             if defined($snapname);
4843
4844         my $size = $drive->{size};
4845         $volhash->{$volid}->{size} //= $size if $size;
4846
4847         $volhash->{$volid}->{is_vmstate} //= 0;
4848         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4849
4850         $volhash->{$volid}->{is_tpmstate} //= 0;
4851         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4852
4853         $volhash->{$volid}->{is_unused} //= 0;
4854         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4855
4856         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4857     };
4858
4859     my $include_opts = {
4860         extra_keys => ['vmstate'],
4861         include_unused => 1,
4862     };
4863
4864     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4865     foreach my $snapname (keys %{$conf->{snapshots}}) {
4866         my $snap = $conf->{snapshots}->{$snapname};
4867         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4868     }
4869
4870     foreach my $volid (keys %$volhash) {
4871         &$func($volid, $volhash->{$volid}, @param);
4872     }
4873 }
4874
4875 my $fast_plug_option = {
4876     'lock' => 1,
4877     'name' => 1,
4878     'onboot' => 1,
4879     'shares' => 1,
4880     'startup' => 1,
4881     'description' => 1,
4882     'protection' => 1,
4883     'vmstatestorage' => 1,
4884     'hookscript' => 1,
4885     'tags' => 1,
4886 };
4887
4888 for my $opt (keys %$confdesc_cloudinit) {
4889     $fast_plug_option->{$opt} = 1;
4890 };
4891
4892 # hotplug changes in [PENDING]
4893 # $selection hash can be used to only apply specified options, for
4894 # example: { cores => 1 } (only apply changed 'cores')
4895 # $errors ref is used to return error messages
4896 sub vmconfig_hotplug_pending {
4897     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4898
4899     my $defaults = load_defaults();
4900     my $arch = get_vm_arch($conf);
4901     my $machine_type = get_vm_machine($conf, undef, $arch);
4902
4903     # commit values which do not have any impact on running VM first
4904     # Note: those option cannot raise errors, we we do not care about
4905     # $selection and always apply them.
4906
4907     my $add_error = sub {
4908         my ($opt, $msg) = @_;
4909         $errors->{$opt} = "hotplug problem - $msg";
4910     };
4911
4912     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4913
4914     my $cloudinit_record_changed = sub {
4915         my ($conf, $opt, $old, $new) = @_;
4916         return if !$cloudinit_pending_properties->{$opt};
4917
4918         my $ci = ($conf->{cloudinit} //= {});
4919
4920         my $recorded = $ci->{$opt};
4921         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4922
4923         if (defined($new)) {
4924             if (defined($old)) {
4925                 # an existing value is being modified
4926                 if (defined($recorded)) {
4927                     # the value was already not in sync
4928                     if ($new eq $recorded) {
4929                         # a value is being reverted to the cloud-init state:
4930                         delete $ci->{$opt};
4931                         delete $added{$opt};
4932                     } else {
4933                         # the value was changed multiple times, do nothing
4934                     }
4935                 } elsif ($added{$opt}) {
4936                     # the value had been marked as added and is being changed, do nothing
4937                 } else {
4938                     # the value is new, record it:
4939                     $ci->{$opt} = $old;
4940                 }
4941             } else {
4942                 # a new value is being added
4943                 if (defined($recorded)) {
4944                     # it was already not in sync
4945                     if ($new eq $recorded) {
4946                         # a value is being reverted to the cloud-init state:
4947                         delete $ci->{$opt};
4948                         delete $added{$opt};
4949                     } else {
4950                         # the value had temporarily been removed, do nothing
4951                     }
4952                 } elsif ($added{$opt}) {
4953                     # the value had been marked as added already, do nothing
4954                 } else {
4955                     # the value is new, add it
4956                     $added{$opt} = 1;
4957                 }
4958             }
4959         } elsif (!defined($old)) {
4960             # a non-existent value is being removed? ignore...
4961         } else {
4962             # a value is being deleted
4963             if (defined($recorded)) {
4964                 # a value was already recorded, just keep it
4965             } elsif ($added{$opt}) {
4966                 # the value was marked as added, remove it
4967                 delete $added{$opt};
4968             } else {
4969                 # a previously unrecorded value is being removed, record the old value:
4970                 $ci->{$opt} = $old;
4971             }
4972         }
4973
4974         my $added = join(',', sort keys %added);
4975         $ci->{added} = $added if length($added);
4976     };
4977
4978     my $changes = 0;
4979     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4980         if ($fast_plug_option->{$opt}) {
4981             my $new = delete $conf->{pending}->{$opt};
4982             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4983             $conf->{$opt} = $new;
4984             $changes = 1;
4985         }
4986     }
4987
4988     if ($changes) {
4989         PVE::QemuConfig->write_config($vmid, $conf);
4990     }
4991
4992     my $ostype = $conf->{ostype};
4993     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4994     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4995     my $usb_hotplug = $hotplug_features->{usb}
4996         && min_version($version, 7, 1)
4997         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4998
4999     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5000     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5001
5002     foreach my $opt (sort keys %$pending_delete_hash) {
5003         next if $selection && !$selection->{$opt};
5004         my $force = $pending_delete_hash->{$opt}->{force};
5005         eval {
5006             if ($opt eq 'hotplug') {
5007                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5008             } elsif ($opt eq 'tablet') {
5009                 die "skip\n" if !$hotplug_features->{usb};
5010                 if ($defaults->{tablet}) {
5011                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5012                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5013                         if $arch eq 'aarch64';
5014                 } else {
5015                     vm_deviceunplug($vmid, $conf, 'tablet');
5016                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5017                 }
5018             } elsif ($opt =~ m/^usb(\d+)$/) {
5019                 my $index = $1;
5020                 die "skip\n" if !$usb_hotplug;
5021                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5022                 vm_deviceunplug($vmid, $conf, $opt);
5023             } elsif ($opt eq 'vcpus') {
5024                 die "skip\n" if !$hotplug_features->{cpu};
5025                 qemu_cpu_hotplug($vmid, $conf, undef);
5026             } elsif ($opt eq 'balloon') {
5027                 # enable balloon device is not hotpluggable
5028                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5029                 # here we reset the ballooning value to memory
5030                 my $balloon = $conf->{memory} || $defaults->{memory};
5031                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5032             } elsif ($fast_plug_option->{$opt}) {
5033                 # do nothing
5034             } elsif ($opt =~ m/^net(\d+)$/) {
5035                 die "skip\n" if !$hotplug_features->{network};
5036                 vm_deviceunplug($vmid, $conf, $opt);
5037             } elsif (is_valid_drivename($opt)) {
5038                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5039                 vm_deviceunplug($vmid, $conf, $opt);
5040                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5041             } elsif ($opt =~ m/^memory$/) {
5042                 die "skip\n" if !$hotplug_features->{memory};
5043                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
5044             } elsif ($opt eq 'cpuunits') {
5045                 $cgroup->change_cpu_shares(undef);
5046             } elsif ($opt eq 'cpulimit') {
5047                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5048             } else {
5049                 die "skip\n";
5050             }
5051         };
5052         if (my $err = $@) {
5053             &$add_error($opt, $err) if $err ne "skip\n";
5054         } else {
5055             my $old = delete $conf->{$opt};
5056             $cloudinit_record_changed->($conf, $opt, $old, undef);
5057             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5058         }
5059     }
5060
5061     my $cloudinit_opt;
5062     foreach my $opt (keys %{$conf->{pending}}) {
5063         next if $selection && !$selection->{$opt};
5064         my $value = $conf->{pending}->{$opt};
5065         eval {
5066             if ($opt eq 'hotplug') {
5067                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5068             } elsif ($opt eq 'tablet') {
5069                 die "skip\n" if !$hotplug_features->{usb};
5070                 if ($value == 1) {
5071                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5072                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5073                         if $arch eq 'aarch64';
5074                 } elsif ($value == 0) {
5075                     vm_deviceunplug($vmid, $conf, 'tablet');
5076                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5077                 }
5078             } elsif ($opt =~ m/^usb(\d+)$/) {
5079                 my $index = $1;
5080                 die "skip\n" if !$usb_hotplug;
5081                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5082                 my $id = $opt;
5083                 if ($d->{host} eq 'spice')  {
5084                     $id = "usbredirdev$index";
5085                 }
5086                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5087             } elsif ($opt eq 'vcpus') {
5088                 die "skip\n" if !$hotplug_features->{cpu};
5089                 qemu_cpu_hotplug($vmid, $conf, $value);
5090             } elsif ($opt eq 'balloon') {
5091                 # enable/disable balloning device is not hotpluggable
5092                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5093                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5094                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5095
5096                 # allow manual ballooning if shares is set to zero
5097                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5098                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5099                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5100                 }
5101             } elsif ($opt =~ m/^net(\d+)$/) {
5102                 # some changes can be done without hotplug
5103                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5104                                     $vmid, $opt, $value, $arch, $machine_type);
5105             } elsif (is_valid_drivename($opt)) {
5106                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5107                 # some changes can be done without hotplug
5108                 my $drive = parse_drive($opt, $value);
5109                 if (drive_is_cloudinit($drive)) {
5110                     $cloudinit_opt = [$opt, $drive];
5111                     # apply all the other changes first, then generate the cloudinit disk
5112                     die "skip\n";
5113                 }
5114                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5115                                      $vmid, $opt, $value, $arch, $machine_type);
5116             } elsif ($opt =~ m/^memory$/) { #dimms
5117                 die "skip\n" if !$hotplug_features->{memory};
5118                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5119             } elsif ($opt eq 'cpuunits') {
5120                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5121                 $cgroup->change_cpu_shares($new_cpuunits);
5122             } elsif ($opt eq 'cpulimit') {
5123                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5124                 $cgroup->change_cpu_quota($cpulimit, 100000);
5125             } elsif ($opt eq 'agent') {
5126                 vmconfig_update_agent($conf, $opt, $value);
5127             } else {
5128                 die "skip\n";  # skip non-hot-pluggable options
5129             }
5130         };
5131         if (my $err = $@) {
5132             &$add_error($opt, $err) if $err ne "skip\n";
5133         } else {
5134             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5135             $conf->{$opt} = $value;
5136             delete $conf->{pending}->{$opt};
5137         }
5138     }
5139
5140     if (defined($cloudinit_opt)) {
5141         my ($opt, $drive) = @$cloudinit_opt;
5142         my $value = $conf->{pending}->{$opt};
5143         eval {
5144             my $temp = {%$conf, $opt => $value};
5145             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5146             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5147                                  $vmid, $opt, $value, $arch, $machine_type);
5148         };
5149         if (my $err = $@) {
5150             &$add_error($opt, $err) if $err ne "skip\n";
5151         } else {
5152             $conf->{$opt} = $value;
5153             delete $conf->{pending}->{$opt};
5154         }
5155     }
5156
5157     # unplug xhci controller if no usb device is left
5158     if ($usb_hotplug) {
5159         my $has_usb = 0;
5160         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5161             next if !defined($conf->{"usb$i"});
5162             $has_usb = 1;
5163             last;
5164         }
5165         if (!$has_usb) {
5166             vm_deviceunplug($vmid, $conf, 'xhci');
5167         }
5168     }
5169
5170     PVE::QemuConfig->write_config($vmid, $conf);
5171
5172     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5173         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5174     }
5175 }
5176
5177 sub try_deallocate_drive {
5178     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5179
5180     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5181         my $volid = $drive->{file};
5182         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5183             my $sid = PVE::Storage::parse_volume_id($volid);
5184             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5185
5186             # check if the disk is really unused
5187             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5188                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5189             PVE::Storage::vdisk_free($storecfg, $volid);
5190             return 1;
5191         } else {
5192             # If vm is not owner of this disk remove from config
5193             return 1;
5194         }
5195     }
5196
5197     return;
5198 }
5199
5200 sub vmconfig_delete_or_detach_drive {
5201     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5202
5203     my $drive = parse_drive($opt, $conf->{$opt});
5204
5205     my $rpcenv = PVE::RPCEnvironment::get();
5206     my $authuser = $rpcenv->get_user();
5207
5208     if ($force) {
5209         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5210         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5211     } else {
5212         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5213     }
5214 }
5215
5216
5217
5218 sub vmconfig_apply_pending {
5219     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5220
5221     return if !scalar(keys %{$conf->{pending}});
5222
5223     my $add_apply_error = sub {
5224         my ($opt, $msg) = @_;
5225         my $err_msg = "unable to apply pending change $opt : $msg";
5226         $errors->{$opt} = $err_msg;
5227         warn $err_msg;
5228     };
5229
5230     # cold plug
5231
5232     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5233     foreach my $opt (sort keys %$pending_delete_hash) {
5234         my $force = $pending_delete_hash->{$opt}->{force};
5235         eval {
5236             if ($opt =~ m/^unused/) {
5237                 die "internal error";
5238             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5239                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5240             }
5241         };
5242         if (my $err = $@) {
5243             $add_apply_error->($opt, $err);
5244         } else {
5245             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5246             delete $conf->{$opt};
5247         }
5248     }
5249
5250     PVE::QemuConfig->cleanup_pending($conf);
5251
5252     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5253
5254     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5255         next if $opt eq 'delete'; # just to be sure
5256         eval {
5257             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5258                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5259             }
5260         };
5261         if (my $err = $@) {
5262             $add_apply_error->($opt, $err);
5263         } else {
5264
5265             if (is_valid_drivename($opt)) {
5266                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5267                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5268             }
5269
5270             $conf->{$opt} = delete $conf->{pending}->{$opt};
5271         }
5272     }
5273
5274     # write all changes at once to avoid unnecessary i/o
5275     PVE::QemuConfig->write_config($vmid, $conf);
5276     if ($generate_cloudinit) {
5277         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5278             # After successful generation and if there were changes to be applied, update the
5279             # config to drop the {cloudinit} entry.
5280             PVE::QemuConfig->write_config($vmid, $conf);
5281         }
5282     }
5283 }
5284
5285 sub vmconfig_update_net {
5286     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5287
5288     my $newnet = parse_net($value);
5289
5290     if ($conf->{$opt}) {
5291         my $oldnet = parse_net($conf->{$opt});
5292
5293         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5294             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5295             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5296             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5297
5298             # for non online change, we try to hot-unplug
5299             die "skip\n" if !$hotplug;
5300             vm_deviceunplug($vmid, $conf, $opt);
5301         } else {
5302
5303             die "internal error" if $opt !~ m/net(\d+)/;
5304             my $iface = "tap${vmid}i$1";
5305
5306             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5307                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5308                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5309                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5310                 PVE::Network::tap_unplug($iface);
5311
5312                 if ($have_sdn) {
5313                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5314                 } else {
5315                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5316                 }
5317             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5318                 # Rate can be applied on its own but any change above needs to
5319                 # include the rate in tap_plug since OVS resets everything.
5320                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5321             }
5322
5323             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5324                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5325             }
5326
5327             return 1;
5328         }
5329     }
5330
5331     if ($hotplug) {
5332         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5333     } else {
5334         die "skip\n";
5335     }
5336 }
5337
5338 sub vmconfig_update_agent {
5339     my ($conf, $opt, $value) = @_;
5340
5341     die "skip\n" if !$conf->{$opt};
5342
5343     my $hotplug_options = { fstrim_cloned_disks => 1 };
5344
5345     my $old_agent = parse_guest_agent($conf);
5346     my $agent = parse_guest_agent({$opt => $value});
5347
5348     for my $option (keys %$agent) { # added/changed options
5349         next if defined($hotplug_options->{$option});
5350         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5351     }
5352
5353     for my $option (keys %$old_agent) { # removed options
5354         next if defined($hotplug_options->{$option});
5355         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5356     }
5357
5358     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5359 }
5360
5361 sub vmconfig_update_disk {
5362     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5363
5364     my $drive = parse_drive($opt, $value);
5365
5366     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5367         my $media = $drive->{media} || 'disk';
5368         my $oldmedia = $old_drive->{media} || 'disk';
5369         die "unable to change media type\n" if $media ne $oldmedia;
5370
5371         if (!drive_is_cdrom($old_drive)) {
5372
5373             if ($drive->{file} ne $old_drive->{file}) {
5374
5375                 die "skip\n" if !$hotplug;
5376
5377                 # unplug and register as unused
5378                 vm_deviceunplug($vmid, $conf, $opt);
5379                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5380
5381             } else {
5382                 # update existing disk
5383
5384                 # skip non hotpluggable value
5385                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5386                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5387                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5388                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5389                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5390                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5391                     die "skip\n";
5392                 }
5393
5394                 # apply throttle
5395                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5396                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5397                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5398                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5399                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5400                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5401                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5402                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5403                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5404                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5405                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5406                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5407                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5408                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5409                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5410                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5411                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5412                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5413
5414                     qemu_block_set_io_throttle(
5415                         $vmid,"drive-$opt",
5416                         ($drive->{mbps} || 0)*1024*1024,
5417                         ($drive->{mbps_rd} || 0)*1024*1024,
5418                         ($drive->{mbps_wr} || 0)*1024*1024,
5419                          $drive->{iops} || 0,
5420                          $drive->{iops_rd} || 0,
5421                          $drive->{iops_wr} || 0,
5422                         ($drive->{mbps_max} || 0)*1024*1024,
5423                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5424                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5425                          $drive->{iops_max} || 0,
5426                          $drive->{iops_rd_max} || 0,
5427                          $drive->{iops_wr_max} || 0,
5428                          $drive->{bps_max_length} || 1,
5429                          $drive->{bps_rd_max_length} || 1,
5430                          $drive->{bps_wr_max_length} || 1,
5431                          $drive->{iops_max_length} || 1,
5432                          $drive->{iops_rd_max_length} || 1,
5433                          $drive->{iops_wr_max_length} || 1,
5434                     );
5435
5436                 }
5437
5438                 return 1;
5439             }
5440
5441         } else { # cdrom
5442
5443             if ($drive->{file} eq 'none') {
5444                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5445                 if (drive_is_cloudinit($old_drive)) {
5446                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5447                 }
5448             } else {
5449                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5450
5451                 # force eject if locked
5452                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5453
5454                 if ($path) {
5455                     mon_cmd($vmid, "blockdev-change-medium",
5456                         id => "$opt", filename => "$path");
5457                 }
5458             }
5459
5460             return 1;
5461         }
5462     }
5463
5464     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5465     # hotplug new disks
5466     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5467     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5468 }
5469
5470 sub vmconfig_update_cloudinit_drive {
5471     my ($storecfg, $conf, $vmid) = @_;
5472
5473     my $cloudinit_ds = undef;
5474     my $cloudinit_drive = undef;
5475
5476     PVE::QemuConfig->foreach_volume($conf, sub {
5477         my ($ds, $drive) = @_;
5478         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5479             $cloudinit_ds = $ds;
5480             $cloudinit_drive = $drive;
5481         }
5482     });
5483
5484     return if !$cloudinit_drive;
5485
5486     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5487         PVE::QemuConfig->write_config($vmid, $conf);
5488     }
5489
5490     my $running = PVE::QemuServer::check_running($vmid);
5491
5492     if ($running) {
5493         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5494         if ($path) {
5495             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5496             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5497         }
5498     }
5499 }
5500
5501 # called in locked context by incoming migration
5502 sub vm_migrate_get_nbd_disks {
5503     my ($storecfg, $conf, $replicated_volumes) = @_;
5504
5505     my $local_volumes = {};
5506     PVE::QemuConfig->foreach_volume($conf, sub {
5507         my ($ds, $drive) = @_;
5508
5509         return if drive_is_cdrom($drive);
5510         return if $ds eq 'tpmstate0';
5511
5512         my $volid = $drive->{file};
5513
5514         return if !$volid;
5515
5516         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5517
5518         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5519         return if $scfg->{shared};
5520
5521         # replicated disks re-use existing state via bitmap
5522         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5523         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5524     });
5525     return $local_volumes;
5526 }
5527
5528 # called in locked context by incoming migration
5529 sub vm_migrate_alloc_nbd_disks {
5530     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5531
5532     my $nbd = {};
5533     foreach my $opt (sort keys %$source_volumes) {
5534         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5535
5536         if ($use_existing) {
5537             $nbd->{$opt}->{drivestr} = print_drive($drive);
5538             $nbd->{$opt}->{volid} = $volid;
5539             $nbd->{$opt}->{replicated} = 1;
5540             next;
5541         }
5542
5543         # storage mapping + volname = regular migration
5544         # storage mapping + format = remote migration
5545         # order of precedence, filtered by whether storage supports it:
5546         # 1. explicit requested format
5547         # 2. format of current volume
5548         # 3. default format of storage
5549         if (!$storagemap->{identity}) {
5550             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5551             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5552             if (!$format || !grep { $format eq $_ } @$validFormats) {
5553                 if ($volname) {
5554                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5555                     my $fileFormat = qemu_img_format($scfg, $volname);
5556                     $format = $fileFormat
5557                         if grep { $fileFormat eq $_ } @$validFormats;
5558                 }
5559                 $format //= $defFormat;
5560             }
5561         } else {
5562             # can't happen for remote migration, so $volname is always defined
5563             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5564             $format = qemu_img_format($scfg, $volname);
5565         }
5566
5567         my $size = $drive->{size} / 1024;
5568         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5569         my $newdrive = $drive;
5570         $newdrive->{format} = $format;
5571         $newdrive->{file} = $newvolid;
5572         my $drivestr = print_drive($newdrive);
5573         $nbd->{$opt}->{drivestr} = $drivestr;
5574         $nbd->{$opt}->{volid} = $newvolid;
5575     }
5576
5577     return $nbd;
5578 }
5579
5580 # see vm_start_nolock for parameters, additionally:
5581 # migrate_opts:
5582 #   storagemap = parsed storage map for allocating NBD disks
5583 sub vm_start {
5584     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5585
5586     return PVE::QemuConfig->lock_config($vmid, sub {
5587         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5588
5589         die "you can't start a vm if it's a template\n"
5590             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5591
5592         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5593         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5594
5595         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5596
5597         if ($has_backup_lock && $running) {
5598             # a backup is currently running, attempt to start the guest in the
5599             # existing QEMU instance
5600             return vm_resume($vmid);
5601         }
5602
5603         PVE::QemuConfig->check_lock($conf)
5604             if !($params->{skiplock} || $has_suspended_lock);
5605
5606         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5607
5608         die "VM $vmid already running\n" if $running;
5609
5610         if (my $storagemap = $migrate_opts->{storagemap}) {
5611             my $replicated = $migrate_opts->{replicated_volumes};
5612             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5613             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5614
5615             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5616                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5617             }
5618         }
5619
5620         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5621     });
5622 }
5623
5624
5625 # params:
5626 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5627 #   skiplock => 0/1, skip checking for config lock
5628 #   skiptemplate => 0/1, skip checking whether VM is template
5629 #   forcemachine => to force QEMU machine (rollback/migration)
5630 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5631 #   timeout => in seconds
5632 #   paused => start VM in paused state (backup)
5633 #   resume => resume from hibernation
5634 #   pbs-backing => {
5635 #      sata0 => {
5636 #         repository
5637 #         snapshot
5638 #         keyfile
5639 #         archive
5640 #      },
5641 #      virtio2 => ...
5642 #   }
5643 # migrate_opts:
5644 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5645 #   migratedfrom => source node
5646 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5647 #   network => CIDR of migration network
5648 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5649 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5650 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5651 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5652 #       contained in config
5653 sub vm_start_nolock {
5654     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5655
5656     my $statefile = $params->{statefile};
5657     my $resume = $params->{resume};
5658
5659     my $migratedfrom = $migrate_opts->{migratedfrom};
5660     my $migration_type = $migrate_opts->{type};
5661
5662     my $res = {};
5663
5664     # clean up leftover reboot request files
5665     eval { clear_reboot_request($vmid); };
5666     warn $@ if $@;
5667
5668     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5669         vmconfig_apply_pending($vmid, $conf, $storecfg);
5670         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5671     }
5672
5673     # don't regenerate the ISO if the VM is started as part of a live migration
5674     # this way we can reuse the old ISO with the correct config
5675     if (!$migratedfrom) {
5676         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5677             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5678             # $conf->{cloudinit}, so we could just not do this?
5679             # But we do it above, so for now let's be consistent.
5680             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5681         }
5682     }
5683
5684     # override offline migrated volumes, conf is out of date still
5685     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5686         for my $key (sort keys $offline_volumes->%*) {
5687             my $parsed = parse_drive($key, $conf->{$key});
5688             $parsed->{file} = $offline_volumes->{$key};
5689             $conf->{$key} = print_drive($parsed);
5690         }
5691     }
5692
5693     my $defaults = load_defaults();
5694
5695     # set environment variable useful inside network script
5696     # for remote migration the config is available on the target node!
5697     if (!$migrate_opts->{remote_node}) {
5698         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5699     }
5700
5701     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5702
5703     my $forcemachine = $params->{forcemachine};
5704     my $forcecpu = $params->{forcecpu};
5705     if ($resume) {
5706         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5707         $forcemachine = $conf->{runningmachine};
5708         $forcecpu = $conf->{runningcpu};
5709         print "Resuming suspended VM\n";
5710     }
5711
5712     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5713         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5714
5715     my $migration_ip;
5716     my $get_migration_ip = sub {
5717         my ($nodename) = @_;
5718
5719         return $migration_ip if defined($migration_ip);
5720
5721         my $cidr = $migrate_opts->{network};
5722
5723         if (!defined($cidr)) {
5724             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5725             $cidr = $dc_conf->{migration}->{network};
5726         }
5727
5728         if (defined($cidr)) {
5729             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5730
5731             die "could not get IP: no address configured on local " .
5732                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5733
5734             die "could not get IP: multiple addresses configured on local " .
5735                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5736
5737             $migration_ip = @$ips[0];
5738         }
5739
5740         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5741             if !defined($migration_ip);
5742
5743         return $migration_ip;
5744     };
5745
5746     if ($statefile) {
5747         if ($statefile eq 'tcp') {
5748             my $migrate = $res->{migrate} = { proto => 'tcp' };
5749             $migrate->{addr} = "localhost";
5750             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5751             my $nodename = nodename();
5752
5753             if (!defined($migration_type)) {
5754                 if (defined($datacenterconf->{migration}->{type})) {
5755                     $migration_type = $datacenterconf->{migration}->{type};
5756                 } else {
5757                     $migration_type = 'secure';
5758                 }
5759             }
5760
5761             if ($migration_type eq 'insecure') {
5762                 $migrate->{addr} = $get_migration_ip->($nodename);
5763                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5764             }
5765
5766             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5767             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5768             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5769             push @$cmd, '-incoming', $migrate->{uri};
5770             push @$cmd, '-S';
5771
5772         } elsif ($statefile eq 'unix') {
5773             # should be default for secure migrations as a ssh TCP forward
5774             # tunnel is not deterministic reliable ready and fails regurarly
5775             # to set up in time, so use UNIX socket forwards
5776             my $migrate = $res->{migrate} = { proto => 'unix' };
5777             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5778             unlink $migrate->{addr};
5779
5780             $migrate->{uri} = "unix:$migrate->{addr}";
5781             push @$cmd, '-incoming', $migrate->{uri};
5782             push @$cmd, '-S';
5783
5784         } elsif (-e $statefile) {
5785             push @$cmd, '-loadstate', $statefile;
5786         } else {
5787             my $statepath = PVE::Storage::path($storecfg, $statefile);
5788             push @$vollist, $statefile;
5789             push @$cmd, '-loadstate', $statepath;
5790         }
5791     } elsif ($params->{paused}) {
5792         push @$cmd, '-S';
5793     }
5794
5795     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5796
5797     my $pci_devices = {}; # host pci devices
5798     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5799         my $dev = $conf->{"hostpci$i"} or next;
5800         $pci_devices->{$i} = parse_hostpci($dev);
5801     }
5802
5803     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5804     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5805
5806     # map to a flat list of pci ids
5807     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5808
5809     # reserve all PCI IDs before actually doing anything with them
5810     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5811
5812     eval {
5813         my $uuid;
5814         for my $id (sort keys %$pci_devices) {
5815             my $d = $pci_devices->{$id};
5816             for my $dev ($d->{pciid}->@*) {
5817                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5818
5819                 # nvidia grid needs the uuid of the mdev as qemu parameter
5820                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5821                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5822                 }
5823             }
5824         }
5825         push @$cmd, '-uuid', $uuid if defined($uuid);
5826     };
5827     if (my $err = $@) {
5828         eval { cleanup_pci_devices($vmid, $conf) };
5829         warn $@ if $@;
5830         die $err;
5831     }
5832
5833     PVE::Storage::activate_volumes($storecfg, $vollist);
5834
5835     eval {
5836         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5837     };
5838     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5839     # timeout should be more than enough here...
5840     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5841
5842     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5843
5844     my %run_params = (
5845         timeout => $statefile ? undef : $start_timeout,
5846         umask => 0077,
5847         noerr => 1,
5848     );
5849
5850     # when migrating, prefix QEMU output so other side can pick up any
5851     # errors that might occur and show the user
5852     if ($migratedfrom) {
5853         $run_params{quiet} = 1;
5854         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5855     }
5856
5857     my %systemd_properties = (
5858         Slice => 'qemu.slice',
5859         KillMode => 'process',
5860         SendSIGKILL => 0,
5861         TimeoutStopUSec => ULONG_MAX, # infinity
5862     );
5863
5864     if (PVE::CGroup::cgroup_mode() == 2) {
5865         $systemd_properties{CPUWeight} = $cpuunits;
5866     } else {
5867         $systemd_properties{CPUShares} = $cpuunits;
5868     }
5869
5870     if (my $cpulimit = $conf->{cpulimit}) {
5871         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5872     }
5873     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5874
5875     my $run_qemu = sub {
5876         PVE::Tools::run_fork sub {
5877             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5878
5879             my $tpmpid;
5880             if (my $tpm = $conf->{tpmstate0}) {
5881                 # start the TPM emulator so QEMU can connect on start
5882                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5883             }
5884
5885             my $exitcode = run_command($cmd, %run_params);
5886             if ($exitcode) {
5887                 if ($tpmpid) {
5888                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5889                     kill 'TERM', $tpmpid;
5890                 }
5891                 die "QEMU exited with code $exitcode\n";
5892             }
5893         };
5894     };
5895
5896     if ($conf->{hugepages}) {
5897
5898         my $code = sub {
5899             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5900             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5901
5902             PVE::QemuServer::Memory::hugepages_mount();
5903             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5904
5905             eval { $run_qemu->() };
5906             if (my $err = $@) {
5907                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5908                     if !$conf->{keephugepages};
5909                 die $err;
5910             }
5911
5912             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5913                 if !$conf->{keephugepages};
5914         };
5915         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5916
5917     } else {
5918         eval { $run_qemu->() };
5919     }
5920
5921     if (my $err = $@) {
5922         # deactivate volumes if start fails
5923         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5924         warn $@ if $@;
5925         eval { cleanup_pci_devices($vmid, $conf) };
5926         warn $@ if $@;
5927
5928         die "start failed: $err";
5929     }
5930
5931     # re-reserve all PCI IDs now that we can know the actual VM PID
5932     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5933     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5934     warn $@ if $@;
5935
5936     if (defined($res->{migrate})) {
5937         print "migration listens on $res->{migrate}->{uri}\n";
5938     } elsif ($statefile) {
5939         eval { mon_cmd($vmid, "cont"); };
5940         warn $@ if $@;
5941     }
5942
5943     #start nbd server for storage migration
5944     if (my $nbd = $migrate_opts->{nbd}) {
5945         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5946
5947         my $migrate_storage_uri;
5948         # nbd_protocol_version > 0 for unix socket support
5949         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5950             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5951             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5952             $migrate_storage_uri = "nbd:unix:$socket_path";
5953             $res->{migrate}->{unix_sockets} = [$socket_path];
5954         } else {
5955             my $nodename = nodename();
5956             my $localip = $get_migration_ip->($nodename);
5957             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5958             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5959
5960             mon_cmd($vmid, "nbd-server-start", addr => {
5961                 type => 'inet',
5962                 data => {
5963                     host => "${localip}",
5964                     port => "${storage_migrate_port}",
5965                 },
5966             });
5967             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5968             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5969         }
5970
5971         foreach my $opt (sort keys %$nbd) {
5972             my $drivestr = $nbd->{$opt}->{drivestr};
5973             my $volid = $nbd->{$opt}->{volid};
5974             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5975             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5976             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5977             print "re-using replicated volume: $opt - $volid\n"
5978                 if $nbd->{$opt}->{replicated};
5979
5980             $res->{drives}->{$opt} = $nbd->{$opt};
5981             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5982         }
5983     }
5984
5985     if ($migratedfrom) {
5986         eval {
5987             set_migration_caps($vmid);
5988         };
5989         warn $@ if $@;
5990
5991         if ($spice_port) {
5992             print "spice listens on port $spice_port\n";
5993             $res->{spice_port} = $spice_port;
5994             if ($migrate_opts->{spice_ticket}) {
5995                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5996                     $migrate_opts->{spice_ticket});
5997                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5998             }
5999         }
6000
6001     } else {
6002         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6003             if !$statefile && $conf->{balloon};
6004
6005         foreach my $opt (keys %$conf) {
6006             next if $opt !~  m/^net\d+$/;
6007             my $nicconf = parse_net($conf->{$opt});
6008             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6009         }
6010         add_nets_bridge_fdb($conf, $vmid);
6011     }
6012
6013     mon_cmd($vmid, 'qom-set',
6014                 path => "machine/peripheral/balloon0",
6015                 property => "guest-stats-polling-interval",
6016                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
6017
6018     if ($resume) {
6019         print "Resumed VM, removing state\n";
6020         if (my $vmstate = $conf->{vmstate}) {
6021             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6022             PVE::Storage::vdisk_free($storecfg, $vmstate);
6023         }
6024         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6025         PVE::QemuConfig->write_config($vmid, $conf);
6026     }
6027
6028     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6029
6030     return $res;
6031 }
6032
6033 sub vm_commandline {
6034     my ($storecfg, $vmid, $snapname) = @_;
6035
6036     my $conf = PVE::QemuConfig->load_config($vmid);
6037
6038     my ($forcemachine, $forcecpu);
6039     if ($snapname) {
6040         my $snapshot = $conf->{snapshots}->{$snapname};
6041         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6042
6043         # check for machine or CPU overrides in snapshot
6044         $forcemachine = $snapshot->{runningmachine};
6045         $forcecpu = $snapshot->{runningcpu};
6046
6047         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6048
6049         $conf = $snapshot;
6050     }
6051
6052     my $defaults = load_defaults();
6053
6054     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6055
6056     return PVE::Tools::cmd2string($cmd);
6057 }
6058
6059 sub vm_reset {
6060     my ($vmid, $skiplock) = @_;
6061
6062     PVE::QemuConfig->lock_config($vmid, sub {
6063
6064         my $conf = PVE::QemuConfig->load_config($vmid);
6065
6066         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6067
6068         mon_cmd($vmid, "system_reset");
6069     });
6070 }
6071
6072 sub get_vm_volumes {
6073     my ($conf) = @_;
6074
6075     my $vollist = [];
6076     foreach_volid($conf, sub {
6077         my ($volid, $attr) = @_;
6078
6079         return if $volid =~ m|^/|;
6080
6081         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6082         return if !$sid;
6083
6084         push @$vollist, $volid;
6085     });
6086
6087     return $vollist;
6088 }
6089
6090 sub cleanup_pci_devices {
6091     my ($vmid, $conf) = @_;
6092
6093     foreach my $key (keys %$conf) {
6094         next if $key !~ m/^hostpci(\d+)$/;
6095         my $hostpciindex = $1;
6096         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6097         my $d = parse_hostpci($conf->{$key});
6098         if ($d->{mdev}) {
6099             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6100             # don't want to break ABI just for this two liner
6101             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6102             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6103         }
6104     }
6105     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6106 }
6107
6108 sub vm_stop_cleanup {
6109     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6110
6111     eval {
6112
6113         if (!$keepActive) {
6114             my $vollist = get_vm_volumes($conf);
6115             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6116
6117             if (my $tpmdrive = $conf->{tpmstate0}) {
6118                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6119                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6120                 if ($storeid) {
6121                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6122                 }
6123             }
6124         }
6125
6126         foreach my $ext (qw(mon qmp pid vnc qga)) {
6127             unlink "/var/run/qemu-server/${vmid}.$ext";
6128         }
6129
6130         if ($conf->{ivshmem}) {
6131             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6132             # just delete it for now, VMs which have this already open do not
6133             # are affected, but new VMs will get a separated one. If this
6134             # becomes an issue we either add some sort of ref-counting or just
6135             # add a "don't delete on stop" flag to the ivshmem format.
6136             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6137         }
6138
6139         cleanup_pci_devices($vmid, $conf);
6140
6141         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6142     };
6143     warn $@ if $@; # avoid errors - just warn
6144 }
6145
6146 # call only in locked context
6147 sub _do_vm_stop {
6148     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6149
6150     my $pid = check_running($vmid, $nocheck);
6151     return if !$pid;
6152
6153     my $conf;
6154     if (!$nocheck) {
6155         $conf = PVE::QemuConfig->load_config($vmid);
6156         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6157         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6158             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6159             $timeout = $opts->{down} if $opts->{down};
6160         }
6161         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6162     }
6163
6164     eval {
6165         if ($shutdown) {
6166             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6167                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6168             } else {
6169                 mon_cmd($vmid, "system_powerdown");
6170             }
6171         } else {
6172             mon_cmd($vmid, "quit");
6173         }
6174     };
6175     my $err = $@;
6176
6177     if (!$err) {
6178         $timeout = 60 if !defined($timeout);
6179
6180         my $count = 0;
6181         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6182             $count++;
6183             sleep 1;
6184         }
6185
6186         if ($count >= $timeout) {
6187             if ($force) {
6188                 warn "VM still running - terminating now with SIGTERM\n";
6189                 kill 15, $pid;
6190             } else {
6191                 die "VM quit/powerdown failed - got timeout\n";
6192             }
6193         } else {
6194             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6195             return;
6196         }
6197     } else {
6198        if (!check_running($vmid, $nocheck)) {
6199             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6200             return;
6201        }
6202        if ($force) {
6203             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6204             kill 15, $pid;
6205         } else {
6206             die "VM quit/powerdown failed\n";
6207         }
6208     }
6209
6210     # wait again
6211     $timeout = 10;
6212
6213     my $count = 0;
6214     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6215         $count++;
6216         sleep 1;
6217     }
6218
6219     if ($count >= $timeout) {
6220         warn "VM still running - terminating now with SIGKILL\n";
6221         kill 9, $pid;
6222         sleep 1;
6223     }
6224
6225     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6226 }
6227
6228 # Note: use $nocheck to skip tests if VM configuration file exists.
6229 # We need that when migration VMs to other nodes (files already moved)
6230 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6231 sub vm_stop {
6232     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6233
6234     $force = 1 if !defined($force) && !$shutdown;
6235
6236     if ($migratedfrom){
6237         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6238         kill 15, $pid if $pid;
6239         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6240         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6241         return;
6242     }
6243
6244     PVE::QemuConfig->lock_config($vmid, sub {
6245         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6246    });
6247 }
6248
6249 sub vm_reboot {
6250     my ($vmid, $timeout) = @_;
6251
6252     PVE::QemuConfig->lock_config($vmid, sub {
6253         eval {
6254
6255             # only reboot if running, as qmeventd starts it again on a stop event
6256             return if !check_running($vmid);
6257
6258             create_reboot_request($vmid);
6259
6260             my $storecfg = PVE::Storage::config();
6261             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6262
6263         };
6264         if (my $err = $@) {
6265             # avoid that the next normal shutdown will be confused for a reboot
6266             clear_reboot_request($vmid);
6267             die $err;
6268         }
6269    });
6270 }
6271
6272 # note: if using the statestorage parameter, the caller has to check privileges
6273 sub vm_suspend {
6274     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6275
6276     my $conf;
6277     my $path;
6278     my $storecfg;
6279     my $vmstate;
6280
6281     PVE::QemuConfig->lock_config($vmid, sub {
6282
6283         $conf = PVE::QemuConfig->load_config($vmid);
6284
6285         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6286         PVE::QemuConfig->check_lock($conf)
6287             if !($skiplock || $is_backing_up);
6288
6289         die "cannot suspend to disk during backup\n"
6290             if $is_backing_up && $includestate;
6291
6292         if ($includestate) {
6293             $conf->{lock} = 'suspending';
6294             my $date = strftime("%Y-%m-%d", localtime(time()));
6295             $storecfg = PVE::Storage::config();
6296             if (!$statestorage) {
6297                 $statestorage = find_vmstate_storage($conf, $storecfg);
6298                 # check permissions for the storage
6299                 my $rpcenv = PVE::RPCEnvironment::get();
6300                 if ($rpcenv->{type} ne 'cli') {
6301                     my $authuser = $rpcenv->get_user();
6302                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6303                 }
6304             }
6305
6306
6307             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6308                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6309             $path = PVE::Storage::path($storecfg, $vmstate);
6310             PVE::QemuConfig->write_config($vmid, $conf);
6311         } else {
6312             mon_cmd($vmid, "stop");
6313         }
6314     });
6315
6316     if ($includestate) {
6317         # save vm state
6318         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6319
6320         eval {
6321             set_migration_caps($vmid, 1);
6322             mon_cmd($vmid, "savevm-start", statefile => $path);
6323             for(;;) {
6324                 my $state = mon_cmd($vmid, "query-savevm");
6325                 if (!$state->{status}) {
6326                     die "savevm not active\n";
6327                 } elsif ($state->{status} eq 'active') {
6328                     sleep(1);
6329                     next;
6330                 } elsif ($state->{status} eq 'completed') {
6331                     print "State saved, quitting\n";
6332                     last;
6333                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6334                     die "query-savevm failed with error '$state->{error}'\n"
6335                 } else {
6336                     die "query-savevm returned status '$state->{status}'\n";
6337                 }
6338             }
6339         };
6340         my $err = $@;
6341
6342         PVE::QemuConfig->lock_config($vmid, sub {
6343             $conf = PVE::QemuConfig->load_config($vmid);
6344             if ($err) {
6345                 # cleanup, but leave suspending lock, to indicate something went wrong
6346                 eval {
6347                     mon_cmd($vmid, "savevm-end");
6348                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6349                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6350                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6351                     PVE::QemuConfig->write_config($vmid, $conf);
6352                 };
6353                 warn $@ if $@;
6354                 die $err;
6355             }
6356
6357             die "lock changed unexpectedly\n"
6358                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6359
6360             mon_cmd($vmid, "quit");
6361             $conf->{lock} = 'suspended';
6362             PVE::QemuConfig->write_config($vmid, $conf);
6363         });
6364     }
6365 }
6366
6367 # $nocheck is set when called as part of a migration - in this context the
6368 # location of the config file (source or target node) is not deterministic,
6369 # since migration cannot wait for pmxcfs to process the rename
6370 sub vm_resume {
6371     my ($vmid, $skiplock, $nocheck) = @_;
6372
6373     PVE::QemuConfig->lock_config($vmid, sub {
6374         my $res = mon_cmd($vmid, 'query-status');
6375         my $resume_cmd = 'cont';
6376         my $reset = 0;
6377         my $conf;
6378         if ($nocheck) {
6379             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6380             if ($@) {
6381                 my $vmlist = PVE::Cluster::get_vmlist();
6382                 if (exists($vmlist->{ids}->{$vmid})) {
6383                     my $node = $vmlist->{ids}->{$vmid}->{node};
6384                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6385                 }
6386                 if (!$conf) {
6387                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6388                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6389                 }
6390             }
6391         } else {
6392             $conf = PVE::QemuConfig->load_config($vmid);
6393         }
6394
6395         if ($res->{status}) {
6396             return if $res->{status} eq 'running'; # job done, go home
6397             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6398             $reset = 1 if $res->{status} eq 'shutdown';
6399         }
6400
6401         if (!$nocheck) {
6402             PVE::QemuConfig->check_lock($conf)
6403                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6404         }
6405
6406         if ($reset) {
6407             # required if a VM shuts down during a backup and we get a resume
6408             # request before the backup finishes for example
6409             mon_cmd($vmid, "system_reset");
6410         }
6411
6412         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6413
6414         mon_cmd($vmid, $resume_cmd);
6415     });
6416 }
6417
6418 sub vm_sendkey {
6419     my ($vmid, $skiplock, $key) = @_;
6420
6421     PVE::QemuConfig->lock_config($vmid, sub {
6422
6423         my $conf = PVE::QemuConfig->load_config($vmid);
6424
6425         # there is no qmp command, so we use the human monitor command
6426         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6427         die $res if $res ne '';
6428     });
6429 }
6430
6431 # vzdump restore implementaion
6432
6433 sub tar_archive_read_firstfile {
6434     my $archive = shift;
6435
6436     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6437
6438     # try to detect archive type first
6439     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6440         die "unable to open file '$archive'\n";
6441     my $firstfile = <$fh>;
6442     kill 15, $pid;
6443     close $fh;
6444
6445     die "ERROR: archive contaions no data\n" if !$firstfile;
6446     chomp $firstfile;
6447
6448     return $firstfile;
6449 }
6450
6451 sub tar_restore_cleanup {
6452     my ($storecfg, $statfile) = @_;
6453
6454     print STDERR "starting cleanup\n";
6455
6456     if (my $fd = IO::File->new($statfile, "r")) {
6457         while (defined(my $line = <$fd>)) {
6458             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6459                 my $volid = $2;
6460                 eval {
6461                     if ($volid =~ m|^/|) {
6462                         unlink $volid || die 'unlink failed\n';
6463                     } else {
6464                         PVE::Storage::vdisk_free($storecfg, $volid);
6465                     }
6466                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6467                 };
6468                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6469             } else {
6470                 print STDERR "unable to parse line in statfile - $line";
6471             }
6472         }
6473         $fd->close();
6474     }
6475 }
6476
6477 sub restore_file_archive {
6478     my ($archive, $vmid, $user, $opts) = @_;
6479
6480     return restore_vma_archive($archive, $vmid, $user, $opts)
6481         if $archive eq '-';
6482
6483     my $info = PVE::Storage::archive_info($archive);
6484     my $format = $opts->{format} // $info->{format};
6485     my $comp = $info->{compression};
6486
6487     # try to detect archive format
6488     if ($format eq 'tar') {
6489         return restore_tar_archive($archive, $vmid, $user, $opts);
6490     } else {
6491         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6492     }
6493 }
6494
6495 # hepler to remove disks that will not be used after restore
6496 my $restore_cleanup_oldconf = sub {
6497     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6498
6499     my $kept_disks = {};
6500
6501     PVE::QemuConfig->foreach_volume($oldconf, sub {
6502         my ($ds, $drive) = @_;
6503
6504         return if drive_is_cdrom($drive, 1);
6505
6506         my $volid = $drive->{file};
6507         return if !$volid || $volid =~ m|^/|;
6508
6509         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6510         return if !$path || !$owner || ($owner != $vmid);
6511
6512         # Note: only delete disk we want to restore
6513         # other volumes will become unused
6514         if ($virtdev_hash->{$ds}) {
6515             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6516             if (my $err = $@) {
6517                 warn $err;
6518             }
6519         } else {
6520             $kept_disks->{$volid} = 1;
6521         }
6522     });
6523
6524     # after the restore we have no snapshots anymore
6525     for my $snapname (keys $oldconf->{snapshots}->%*) {
6526         my $snap = $oldconf->{snapshots}->{$snapname};
6527         if ($snap->{vmstate}) {
6528             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6529             if (my $err = $@) {
6530                 warn $err;
6531             }
6532         }
6533
6534         for my $volid (keys $kept_disks->%*) {
6535             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6536             warn $@ if $@;
6537         }
6538     }
6539 };
6540
6541 # Helper to parse vzdump backup device hints
6542 #
6543 # $rpcenv: Environment, used to ckeck storage permissions
6544 # $user: User ID, to check storage permissions
6545 # $storecfg: Storage configuration
6546 # $fh: the file handle for reading the configuration
6547 # $devinfo: should contain device sizes for all backu-up'ed devices
6548 # $options: backup options (pool, default storage)
6549 #
6550 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6551 my $parse_backup_hints = sub {
6552     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6553
6554     my $check_storage = sub { # assert if an image can be allocate
6555         my ($storeid, $scfg) = @_;
6556         die "Content type 'images' is not available on storage '$storeid'\n"
6557             if !$scfg->{content}->{images};
6558         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6559             if $user ne 'root@pam';
6560     };
6561
6562     my $virtdev_hash = {};
6563     while (defined(my $line = <$fh>)) {
6564         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6565             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6566             die "archive does not contain data for drive '$virtdev'\n"
6567                 if !$devinfo->{$devname};
6568
6569             if (defined($options->{storage})) {
6570                 $storeid = $options->{storage} || 'local';
6571             } elsif (!$storeid) {
6572                 $storeid = 'local';
6573             }
6574             $format = 'raw' if !$format;
6575             $devinfo->{$devname}->{devname} = $devname;
6576             $devinfo->{$devname}->{virtdev} = $virtdev;
6577             $devinfo->{$devname}->{format} = $format;
6578             $devinfo->{$devname}->{storeid} = $storeid;
6579
6580             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6581             $check_storage->($storeid, $scfg); # permission and content type check
6582
6583             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6584         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6585             my $virtdev = $1;
6586             my $drive = parse_drive($virtdev, $2);
6587
6588             if (drive_is_cloudinit($drive)) {
6589                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6590                 $storeid = $options->{storage} if defined ($options->{storage});
6591                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6592                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6593
6594                 $check_storage->($storeid, $scfg); # permission and content type check
6595
6596                 $virtdev_hash->{$virtdev} = {
6597                     format => $format,
6598                     storeid => $storeid,
6599                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6600                     is_cloudinit => 1,
6601                 };
6602             }
6603         }
6604     }
6605
6606     return $virtdev_hash;
6607 };
6608
6609 # Helper to allocate and activate all volumes required for a restore
6610 #
6611 # $storecfg: Storage configuration
6612 # $virtdev_hash: as returned by parse_backup_hints()
6613 #
6614 # Returns: { $virtdev => $volid }
6615 my $restore_allocate_devices = sub {
6616     my ($storecfg, $virtdev_hash, $vmid) = @_;
6617
6618     my $map = {};
6619     foreach my $virtdev (sort keys %$virtdev_hash) {
6620         my $d = $virtdev_hash->{$virtdev};
6621         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6622         my $storeid = $d->{storeid};
6623         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6624
6625         # test if requested format is supported
6626         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6627         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6628         $d->{format} = $defFormat if !$supported;
6629
6630         my $name;
6631         if ($d->{is_cloudinit}) {
6632             $name = "vm-$vmid-cloudinit";
6633             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6634             if ($scfg->{path}) {
6635                 $name .= ".$d->{format}";
6636             }
6637         }
6638
6639         my $volid = PVE::Storage::vdisk_alloc(
6640             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6641
6642         print STDERR "new volume ID is '$volid'\n";
6643         $d->{volid} = $volid;
6644
6645         PVE::Storage::activate_volumes($storecfg, [$volid]);
6646
6647         $map->{$virtdev} = $volid;
6648     }
6649
6650     return $map;
6651 };
6652
6653 sub restore_update_config_line {
6654     my ($cookie, $map, $line, $unique) = @_;
6655
6656     return '' if $line =~ m/^\#qmdump\#/;
6657     return '' if $line =~ m/^\#vzdump\#/;
6658     return '' if $line =~ m/^lock:/;
6659     return '' if $line =~ m/^unused\d+:/;
6660     return '' if $line =~ m/^parent:/;
6661
6662     my $res = '';
6663
6664     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6665     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6666         # try to convert old 1.X settings
6667         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6668         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6669             my ($model, $macaddr) = split(/\=/, $devconfig);
6670             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6671             my $net = {
6672                 model => $model,
6673                 bridge => "vmbr$ind",
6674                 macaddr => $macaddr,
6675             };
6676             my $netstr = print_net($net);
6677
6678             $res .= "net$cookie->{netcount}: $netstr\n";
6679             $cookie->{netcount}++;
6680         }
6681     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6682         my ($id, $netstr) = ($1, $2);
6683         my $net = parse_net($netstr);
6684         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6685         $netstr = print_net($net);
6686         $res .= "$id: $netstr\n";
6687     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6688         my $virtdev = $1;
6689         my $value = $3;
6690         my $di = parse_drive($virtdev, $value);
6691         if (defined($di->{backup}) && !$di->{backup}) {
6692             $res .= "#$line";
6693         } elsif ($map->{$virtdev}) {
6694             delete $di->{format}; # format can change on restore
6695             $di->{file} = $map->{$virtdev};
6696             $value = print_drive($di);
6697             $res .= "$virtdev: $value\n";
6698         } else {
6699             $res .= $line;
6700         }
6701     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6702         my $vmgenid = $1;
6703         if ($vmgenid ne '0') {
6704             # always generate a new vmgenid if there was a valid one setup
6705             $vmgenid = generate_uuid();
6706         }
6707         $res .= "vmgenid: $vmgenid\n";
6708     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6709         my ($uuid, $uuid_str);
6710         UUID::generate($uuid);
6711         UUID::unparse($uuid, $uuid_str);
6712         my $smbios1 = parse_smbios1($2);
6713         $smbios1->{uuid} = $uuid_str;
6714         $res .= $1.print_smbios1($smbios1)."\n";
6715     } else {
6716         $res .= $line;
6717     }
6718
6719     return $res;
6720 }
6721
6722 my $restore_deactivate_volumes = sub {
6723     my ($storecfg, $virtdev_hash) = @_;
6724
6725     my $vollist = [];
6726     for my $dev (values $virtdev_hash->%*) {
6727         push $vollist->@*, $dev->{volid} if $dev->{volid};
6728     }
6729
6730     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6731     print STDERR $@ if $@;
6732 };
6733
6734 my $restore_destroy_volumes = sub {
6735     my ($storecfg, $virtdev_hash) = @_;
6736
6737     for my $dev (values $virtdev_hash->%*) {
6738         my $volid = $dev->{volid} or next;
6739         eval {
6740             PVE::Storage::vdisk_free($storecfg, $volid);
6741             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6742         };
6743         print STDERR "unable to cleanup '$volid' - $@" if $@;
6744     }
6745 };
6746
6747 my $restore_merge_config = sub {
6748     my ($filename, $backup_conf_raw, $override_conf) = @_;
6749
6750     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6751     for my $key (keys $override_conf->%*) {
6752         $backup_conf->{$key} = $override_conf->{$key};
6753     }
6754
6755     return $backup_conf;
6756 };
6757
6758 sub scan_volids {
6759     my ($cfg, $vmid) = @_;
6760
6761     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6762
6763     my $volid_hash = {};
6764     foreach my $storeid (keys %$info) {
6765         foreach my $item (@{$info->{$storeid}}) {
6766             next if !($item->{volid} && $item->{size});
6767             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6768             $volid_hash->{$item->{volid}} = $item;
6769         }
6770     }
6771
6772     return $volid_hash;
6773 }
6774
6775 sub update_disk_config {
6776     my ($vmid, $conf, $volid_hash) = @_;
6777
6778     my $changes;
6779     my $prefix = "VM $vmid";
6780
6781     # used and unused disks
6782     my $referenced = {};
6783
6784     # Note: it is allowed to define multiple storages with same path (alias), so
6785     # we need to check both 'volid' and real 'path' (two different volid can point
6786     # to the same path).
6787
6788     my $referencedpath = {};
6789
6790     # update size info
6791     PVE::QemuConfig->foreach_volume($conf, sub {
6792         my ($opt, $drive) = @_;
6793
6794         my $volid = $drive->{file};
6795         return if !$volid;
6796         my $volume = $volid_hash->{$volid};
6797
6798         # mark volid as "in-use" for next step
6799         $referenced->{$volid} = 1;
6800         if ($volume && (my $path = $volume->{path})) {
6801             $referencedpath->{$path} = 1;
6802         }
6803
6804         return if drive_is_cdrom($drive);
6805         return if !$volume;
6806
6807         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6808         if (defined($updated)) {
6809             $changes = 1;
6810             $conf->{$opt} = print_drive($updated);
6811             print "$prefix ($opt): $msg\n";
6812         }
6813     });
6814
6815     # remove 'unusedX' entry if volume is used
6816     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6817         my ($opt, $drive) = @_;
6818
6819         my $volid = $drive->{file};
6820         return if !$volid;
6821
6822         my $path;
6823         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6824         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6825             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6826             $changes = 1;
6827             delete $conf->{$opt};
6828         }
6829
6830         $referenced->{$volid} = 1;
6831         $referencedpath->{$path} = 1 if $path;
6832     });
6833
6834     foreach my $volid (sort keys %$volid_hash) {
6835         next if $volid =~ m/vm-$vmid-state-/;
6836         next if $referenced->{$volid};
6837         my $path = $volid_hash->{$volid}->{path};
6838         next if !$path; # just to be sure
6839         next if $referencedpath->{$path};
6840         $changes = 1;
6841         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6842         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6843         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6844     }
6845
6846     return $changes;
6847 }
6848
6849 sub rescan {
6850     my ($vmid, $nolock, $dryrun) = @_;
6851
6852     my $cfg = PVE::Storage::config();
6853
6854     print "rescan volumes...\n";
6855     my $volid_hash = scan_volids($cfg, $vmid);
6856
6857     my $updatefn =  sub {
6858         my ($vmid) = @_;
6859
6860         my $conf = PVE::QemuConfig->load_config($vmid);
6861
6862         PVE::QemuConfig->check_lock($conf);
6863
6864         my $vm_volids = {};
6865         foreach my $volid (keys %$volid_hash) {
6866             my $info = $volid_hash->{$volid};
6867             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6868         }
6869
6870         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6871
6872         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6873     };
6874
6875     if (defined($vmid)) {
6876         if ($nolock) {
6877             &$updatefn($vmid);
6878         } else {
6879             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6880         }
6881     } else {
6882         my $vmlist = config_list();
6883         foreach my $vmid (keys %$vmlist) {
6884             if ($nolock) {
6885                 &$updatefn($vmid);
6886             } else {
6887                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6888             }
6889         }
6890     }
6891 }
6892
6893 sub restore_proxmox_backup_archive {
6894     my ($archive, $vmid, $user, $options) = @_;
6895
6896     my $storecfg = PVE::Storage::config();
6897
6898     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6899     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6900
6901     my $fingerprint = $scfg->{fingerprint};
6902     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6903
6904     my $repo = PVE::PBSClient::get_repository($scfg);
6905     my $namespace = $scfg->{namespace};
6906
6907     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6908     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6909     local $ENV{PBS_PASSWORD} = $password;
6910     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6911
6912     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6913         PVE::Storage::parse_volname($storecfg, $archive);
6914
6915     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6916
6917     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6918
6919     my $tmpdir = "/var/tmp/vzdumptmp$$";
6920     rmtree $tmpdir;
6921     mkpath $tmpdir;
6922
6923     my $conffile = PVE::QemuConfig->config_file($vmid);
6924      # disable interrupts (always do cleanups)
6925     local $SIG{INT} =
6926         local $SIG{TERM} =
6927         local $SIG{QUIT} =
6928         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6929
6930     # Note: $oldconf is undef if VM does not exists
6931     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6932     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6933     my $new_conf_raw = '';
6934
6935     my $rpcenv = PVE::RPCEnvironment::get();
6936     my $devinfo = {}; # info about drives included in backup
6937     my $virtdev_hash = {}; # info about allocated drives
6938
6939     eval {
6940         # enable interrupts
6941         local $SIG{INT} =
6942             local $SIG{TERM} =
6943             local $SIG{QUIT} =
6944             local $SIG{HUP} =
6945             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6946
6947         my $cfgfn = "$tmpdir/qemu-server.conf";
6948         my $firewall_config_fn = "$tmpdir/fw.conf";
6949         my $index_fn = "$tmpdir/index.json";
6950
6951         my $cmd = "restore";
6952
6953         my $param = [$pbs_backup_name, "index.json", $index_fn];
6954         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6955         my $index = PVE::Tools::file_get_contents($index_fn);
6956         $index = decode_json($index);
6957
6958         foreach my $info (@{$index->{files}}) {
6959             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6960                 my $devname = $1;
6961                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6962                     $devinfo->{$devname}->{size} = $1;
6963                 } else {
6964                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6965                 }
6966             }
6967         }
6968
6969         my $is_qemu_server_backup = scalar(
6970             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6971         );
6972         if (!$is_qemu_server_backup) {
6973             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6974         }
6975         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6976
6977         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6978         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6979
6980         if ($has_firewall_config) {
6981             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6982             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6983
6984             my $pve_firewall_dir = '/etc/pve/firewall';
6985             mkdir $pve_firewall_dir; # make sure the dir exists
6986             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6987         }
6988
6989         my $fh = IO::File->new($cfgfn, "r") ||
6990             die "unable to read qemu-server.conf - $!\n";
6991
6992         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6993
6994         # fixme: rate limit?
6995
6996         # create empty/temp config
6997         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6998
6999         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7000
7001         # allocate volumes
7002         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7003
7004         foreach my $virtdev (sort keys %$virtdev_hash) {
7005             my $d = $virtdev_hash->{$virtdev};
7006             next if $d->{is_cloudinit}; # no need to restore cloudinit
7007
7008             # this fails if storage is unavailable
7009             my $volid = $d->{volid};
7010             my $path = PVE::Storage::path($storecfg, $volid);
7011
7012             # for live-restore we only want to preload the efidisk and TPM state
7013             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7014
7015             my @ns_arg;
7016             if (defined(my $ns = $scfg->{namespace})) {
7017                 @ns_arg = ('--ns', $ns);
7018             }
7019
7020             my $pbs_restore_cmd = [
7021                 '/usr/bin/pbs-restore',
7022                 '--repository', $repo,
7023                 @ns_arg,
7024                 $pbs_backup_name,
7025                 "$d->{devname}.img.fidx",
7026                 $path,
7027                 '--verbose',
7028                 ];
7029
7030             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7031             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7032
7033             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7034                 push @$pbs_restore_cmd, '--skip-zero';
7035             }
7036
7037             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7038             print "restore proxmox backup image: $dbg_cmdstring\n";
7039             run_command($pbs_restore_cmd);
7040         }
7041
7042         $fh->seek(0, 0) || die "seek failed - $!\n";
7043
7044         my $cookie = { netcount => 0 };
7045         while (defined(my $line = <$fh>)) {
7046             $new_conf_raw .= restore_update_config_line(
7047                 $cookie,
7048                 $map,
7049                 $line,
7050                 $options->{unique},
7051             );
7052         }
7053
7054         $fh->close();
7055     };
7056     my $err = $@;
7057
7058     if ($err || !$options->{live}) {
7059         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7060     }
7061
7062     rmtree $tmpdir;
7063
7064     if ($err) {
7065         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7066         die $err;
7067     }
7068
7069     if ($options->{live}) {
7070         # keep lock during live-restore
7071         $new_conf_raw .= "\nlock: create";
7072     }
7073
7074     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7075     PVE::QemuConfig->write_config($vmid, $new_conf);
7076
7077     eval { rescan($vmid, 1); };
7078     warn $@ if $@;
7079
7080     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7081
7082     if ($options->{live}) {
7083         # enable interrupts
7084         local $SIG{INT} =
7085             local $SIG{TERM} =
7086             local $SIG{QUIT} =
7087             local $SIG{HUP} =
7088             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7089
7090         my $conf = PVE::QemuConfig->load_config($vmid);
7091         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7092
7093         # these special drives are already restored before start
7094         delete $devinfo->{'drive-efidisk0'};
7095         delete $devinfo->{'drive-tpmstate0-backup'};
7096
7097         my $pbs_opts = {
7098             repo => $repo,
7099             keyfile => $keyfile,
7100             snapshot => $pbs_backup_name,
7101             namespace => $namespace,
7102         };
7103         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7104
7105         PVE::QemuConfig->remove_lock($vmid, "create");
7106     }
7107 }
7108
7109 sub pbs_live_restore {
7110     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7111
7112     print "starting VM for live-restore\n";
7113     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7114
7115     my $pbs_backing = {};
7116     for my $ds (keys %$restored_disks) {
7117         $ds =~ m/^drive-(.*)$/;
7118         my $confname = $1;
7119         $pbs_backing->{$confname} = {
7120             repository => $opts->{repo},
7121             snapshot => $opts->{snapshot},
7122             archive => "$ds.img.fidx",
7123         };
7124         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7125         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7126
7127         my $drive = parse_drive($confname, $conf->{$confname});
7128         print "restoring '$ds' to '$drive->{file}'\n";
7129     }
7130
7131     my $drives_streamed = 0;
7132     eval {
7133         # make sure HA doesn't interrupt our restore by stopping the VM
7134         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7135             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7136         }
7137
7138         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7139         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7140         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7141
7142         my $qmeventd_fd = register_qmeventd_handle($vmid);
7143
7144         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7145         # this will effectively collapse the backing image chain consisting of
7146         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7147         # removes itself once all backing images vanish with 'auto-remove=on')
7148         my $jobs = {};
7149         for my $ds (sort keys %$restored_disks) {
7150             my $job_id = "restore-$ds";
7151             mon_cmd($vmid, 'block-stream',
7152                 'job-id' => $job_id,
7153                 device => "$ds",
7154             );
7155             $jobs->{$job_id} = {};
7156         }
7157
7158         mon_cmd($vmid, 'cont');
7159         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7160
7161         print "restore-drive jobs finished successfully, removing all tracking block devices"
7162             ." to disconnect from Proxmox Backup Server\n";
7163
7164         for my $ds (sort keys %$restored_disks) {
7165             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7166         }
7167
7168         close($qmeventd_fd);
7169     };
7170
7171     my $err = $@;
7172
7173     if ($err) {
7174         warn "An error occurred during live-restore: $err\n";
7175         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7176         die "live-restore failed\n";
7177     }
7178 }
7179
7180 sub restore_vma_archive {
7181     my ($archive, $vmid, $user, $opts, $comp) = @_;
7182
7183     my $readfrom = $archive;
7184
7185     my $cfg = PVE::Storage::config();
7186     my $commands = [];
7187     my $bwlimit = $opts->{bwlimit};
7188
7189     my $dbg_cmdstring = '';
7190     my $add_pipe = sub {
7191         my ($cmd) = @_;
7192         push @$commands, $cmd;
7193         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7194         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7195         $readfrom = '-';
7196     };
7197
7198     my $input = undef;
7199     if ($archive eq '-') {
7200         $input = '<&STDIN';
7201     } else {
7202         # If we use a backup from a PVE defined storage we also consider that
7203         # storage's rate limit:
7204         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7205         if (defined($volid)) {
7206             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7207             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7208             if ($readlimit) {
7209                 print STDERR "applying read rate limit: $readlimit\n";
7210                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7211                 $add_pipe->($cstream);
7212             }
7213         }
7214     }
7215
7216     if ($comp) {
7217         my $info = PVE::Storage::decompressor_info('vma', $comp);
7218         my $cmd = $info->{decompressor};
7219         push @$cmd, $readfrom;
7220         $add_pipe->($cmd);
7221     }
7222
7223     my $tmpdir = "/var/tmp/vzdumptmp$$";
7224     rmtree $tmpdir;
7225
7226     # disable interrupts (always do cleanups)
7227     local $SIG{INT} =
7228         local $SIG{TERM} =
7229         local $SIG{QUIT} =
7230         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7231
7232     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7233     POSIX::mkfifo($mapfifo, 0600);
7234     my $fifofh;
7235     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7236
7237     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7238
7239     my $oldtimeout;
7240     my $timeout = 5;
7241
7242     my $devinfo = {}; # info about drives included in backup
7243     my $virtdev_hash = {}; # info about allocated drives
7244
7245     my $rpcenv = PVE::RPCEnvironment::get();
7246
7247     my $conffile = PVE::QemuConfig->config_file($vmid);
7248
7249     # Note: $oldconf is undef if VM does not exist
7250     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7251     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7252     my $new_conf_raw = '';
7253
7254     my %storage_limits;
7255
7256     my $print_devmap = sub {
7257         my $cfgfn = "$tmpdir/qemu-server.conf";
7258
7259         # we can read the config - that is already extracted
7260         my $fh = IO::File->new($cfgfn, "r") ||
7261             die "unable to read qemu-server.conf - $!\n";
7262
7263         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7264         if (-f $fwcfgfn) {
7265             my $pve_firewall_dir = '/etc/pve/firewall';
7266             mkdir $pve_firewall_dir; # make sure the dir exists
7267             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7268         }
7269
7270         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7271
7272         foreach my $info (values %{$virtdev_hash}) {
7273             my $storeid = $info->{storeid};
7274             next if defined($storage_limits{$storeid});
7275
7276             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7277             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7278             $storage_limits{$storeid} = $limit * 1024;
7279         }
7280
7281         foreach my $devname (keys %$devinfo) {
7282             die "found no device mapping information for device '$devname'\n"
7283                 if !$devinfo->{$devname}->{virtdev};
7284         }
7285
7286         # create empty/temp config
7287         if ($oldconf) {
7288             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7289             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7290         }
7291
7292         # allocate volumes
7293         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7294
7295         # print restore information to $fifofh
7296         foreach my $virtdev (sort keys %$virtdev_hash) {
7297             my $d = $virtdev_hash->{$virtdev};
7298             next if $d->{is_cloudinit}; # no need to restore cloudinit
7299
7300             my $storeid = $d->{storeid};
7301             my $volid = $d->{volid};
7302
7303             my $map_opts = '';
7304             if (my $limit = $storage_limits{$storeid}) {
7305                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7306             }
7307
7308             my $write_zeros = 1;
7309             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7310                 $write_zeros = 0;
7311             }
7312
7313             my $path = PVE::Storage::path($cfg, $volid);
7314
7315             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7316
7317             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7318         }
7319
7320         $fh->seek(0, 0) || die "seek failed - $!\n";
7321
7322         my $cookie = { netcount => 0 };
7323         while (defined(my $line = <$fh>)) {
7324             $new_conf_raw .= restore_update_config_line(
7325                 $cookie,
7326                 $map,
7327                 $line,
7328                 $opts->{unique},
7329             );
7330         }
7331
7332         $fh->close();
7333     };
7334
7335     eval {
7336         # enable interrupts
7337         local $SIG{INT} =
7338             local $SIG{TERM} =
7339             local $SIG{QUIT} =
7340             local $SIG{HUP} =
7341             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7342         local $SIG{ALRM} = sub { die "got timeout\n"; };
7343
7344         $oldtimeout = alarm($timeout);
7345
7346         my $parser = sub {
7347             my $line = shift;
7348
7349             print "$line\n";
7350
7351             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7352                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7353                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7354             } elsif ($line =~ m/^CTIME: /) {
7355                 # we correctly received the vma config, so we can disable
7356                 # the timeout now for disk allocation (set to 10 minutes, so
7357                 # that we always timeout if something goes wrong)
7358                 alarm(600);
7359                 &$print_devmap();
7360                 print $fifofh "done\n";
7361                 my $tmp = $oldtimeout || 0;
7362                 $oldtimeout = undef;
7363                 alarm($tmp);
7364                 close($fifofh);
7365                 $fifofh = undef;
7366             }
7367         };
7368
7369         print "restore vma archive: $dbg_cmdstring\n";
7370         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7371     };
7372     my $err = $@;
7373
7374     alarm($oldtimeout) if $oldtimeout;
7375
7376     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7377
7378     close($fifofh) if $fifofh;
7379     unlink $mapfifo;
7380     rmtree $tmpdir;
7381
7382     if ($err) {
7383         $restore_destroy_volumes->($cfg, $virtdev_hash);
7384         die $err;
7385     }
7386
7387     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7388     PVE::QemuConfig->write_config($vmid, $new_conf);
7389
7390     eval { rescan($vmid, 1); };
7391     warn $@ if $@;
7392
7393     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7394 }
7395
7396 sub restore_tar_archive {
7397     my ($archive, $vmid, $user, $opts) = @_;
7398
7399     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7400         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7401         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7402     }
7403
7404     if ($archive ne '-') {
7405         my $firstfile = tar_archive_read_firstfile($archive);
7406         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7407             if $firstfile ne 'qemu-server.conf';
7408     }
7409
7410     my $storecfg = PVE::Storage::config();
7411
7412     # avoid zombie disks when restoring over an existing VM -> cleanup first
7413     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7414     # skiplock=1 because qmrestore has set the 'create' lock itself already
7415     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7416     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7417
7418     my $tocmd = "/usr/lib/qemu-server/qmextract";
7419
7420     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7421     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7422     $tocmd .= ' --prealloc' if $opts->{prealloc};
7423     $tocmd .= ' --info' if $opts->{info};
7424
7425     # tar option "xf" does not autodetect compression when read from STDIN,
7426     # so we pipe to zcat
7427     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7428         PVE::Tools::shellquote("--to-command=$tocmd");
7429
7430     my $tmpdir = "/var/tmp/vzdumptmp$$";
7431     mkpath $tmpdir;
7432
7433     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7434     local $ENV{VZDUMP_VMID} = $vmid;
7435     local $ENV{VZDUMP_USER} = $user;
7436
7437     my $conffile = PVE::QemuConfig->config_file($vmid);
7438     my $new_conf_raw = '';
7439
7440     # disable interrupts (always do cleanups)
7441     local $SIG{INT} =
7442         local $SIG{TERM} =
7443         local $SIG{QUIT} =
7444         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7445
7446     eval {
7447         # enable interrupts
7448         local $SIG{INT} =
7449             local $SIG{TERM} =
7450             local $SIG{QUIT} =
7451             local $SIG{HUP} =
7452             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7453
7454         if ($archive eq '-') {
7455             print "extracting archive from STDIN\n";
7456             run_command($cmd, input => "<&STDIN");
7457         } else {
7458             print "extracting archive '$archive'\n";
7459             run_command($cmd);
7460         }
7461
7462         return if $opts->{info};
7463
7464         # read new mapping
7465         my $map = {};
7466         my $statfile = "$tmpdir/qmrestore.stat";
7467         if (my $fd = IO::File->new($statfile, "r")) {
7468             while (defined (my $line = <$fd>)) {
7469                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7470                     $map->{$1} = $2 if $1;
7471                 } else {
7472                     print STDERR "unable to parse line in statfile - $line\n";
7473                 }
7474             }
7475             $fd->close();
7476         }
7477
7478         my $confsrc = "$tmpdir/qemu-server.conf";
7479
7480         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7481
7482         my $cookie = { netcount => 0 };
7483         while (defined (my $line = <$srcfd>)) {
7484             $new_conf_raw .= restore_update_config_line(
7485                 $cookie,
7486                 $map,
7487                 $line,
7488                 $opts->{unique},
7489             );
7490         }
7491
7492         $srcfd->close();
7493     };
7494     if (my $err = $@) {
7495         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7496         die $err;
7497     }
7498
7499     rmtree $tmpdir;
7500
7501     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7502
7503     PVE::Cluster::cfs_update(); # make sure we read new file
7504
7505     eval { rescan($vmid, 1); };
7506     warn $@ if $@;
7507 };
7508
7509 sub foreach_storage_used_by_vm {
7510     my ($conf, $func) = @_;
7511
7512     my $sidhash = {};
7513
7514     PVE::QemuConfig->foreach_volume($conf, sub {
7515         my ($ds, $drive) = @_;
7516         return if drive_is_cdrom($drive);
7517
7518         my $volid = $drive->{file};
7519
7520         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7521         $sidhash->{$sid} = $sid if $sid;
7522     });
7523
7524     foreach my $sid (sort keys %$sidhash) {
7525         &$func($sid);
7526     }
7527 }
7528
7529 my $qemu_snap_storage = {
7530     rbd => 1,
7531 };
7532 sub do_snapshots_with_qemu {
7533     my ($storecfg, $volid, $deviceid) = @_;
7534
7535     return if $deviceid =~ m/tpmstate0/;
7536
7537     my $storage_name = PVE::Storage::parse_volume_id($volid);
7538     my $scfg = $storecfg->{ids}->{$storage_name};
7539     die "could not find storage '$storage_name'\n" if !defined($scfg);
7540
7541     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7542         return 1;
7543     }
7544
7545     if ($volid =~ m/\.(qcow2|qed)$/){
7546         return 1;
7547     }
7548
7549     return;
7550 }
7551
7552 sub qga_check_running {
7553     my ($vmid, $nowarn) = @_;
7554
7555     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7556     if ($@) {
7557         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7558         return 0;
7559     }
7560     return 1;
7561 }
7562
7563 sub template_create {
7564     my ($vmid, $conf, $disk) = @_;
7565
7566     my $storecfg = PVE::Storage::config();
7567
7568     PVE::QemuConfig->foreach_volume($conf, sub {
7569         my ($ds, $drive) = @_;
7570
7571         return if drive_is_cdrom($drive);
7572         return if $disk && $ds ne $disk;
7573
7574         my $volid = $drive->{file};
7575         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7576
7577         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7578         $drive->{file} = $voliddst;
7579         $conf->{$ds} = print_drive($drive);
7580         PVE::QemuConfig->write_config($vmid, $conf);
7581     });
7582 }
7583
7584 sub convert_iscsi_path {
7585     my ($path) = @_;
7586
7587     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7588         my $portal = $1;
7589         my $target = $2;
7590         my $lun = $3;
7591
7592         my $initiator_name = get_initiator_name();
7593
7594         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7595                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7596     }
7597
7598     die "cannot convert iscsi path '$path', unkown format\n";
7599 }
7600
7601 sub qemu_img_convert {
7602     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7603
7604     my $storecfg = PVE::Storage::config();
7605     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7606     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7607
7608     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7609
7610     my $cachemode;
7611     my $src_path;
7612     my $src_is_iscsi = 0;
7613     my $src_format;
7614
7615     if ($src_storeid) {
7616         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7617         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7618         $src_format = qemu_img_format($src_scfg, $src_volname);
7619         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7620         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7621         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7622     } elsif (-f $src_volid || -b $src_volid) {
7623         $src_path = $src_volid;
7624         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7625             $src_format = $1;
7626         }
7627     }
7628
7629     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7630
7631     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7632     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7633     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7634     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7635
7636     my $cmd = [];
7637     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7638     push @$cmd, '-l', "snapshot.name=$snapname"
7639         if $snapname && $src_format && $src_format eq "qcow2";
7640     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7641     push @$cmd, '-T', $cachemode if defined($cachemode);
7642
7643     if ($src_is_iscsi) {
7644         push @$cmd, '--image-opts';
7645         $src_path = convert_iscsi_path($src_path);
7646     } elsif ($src_format) {
7647         push @$cmd, '-f', $src_format;
7648     }
7649
7650     if ($dst_is_iscsi) {
7651         push @$cmd, '--target-image-opts';
7652         $dst_path = convert_iscsi_path($dst_path);
7653     } else {
7654         push @$cmd, '-O', $dst_format;
7655     }
7656
7657     push @$cmd, $src_path;
7658
7659     if (!$dst_is_iscsi && $is_zero_initialized) {
7660         push @$cmd, "zeroinit:$dst_path";
7661     } else {
7662         push @$cmd, $dst_path;
7663     }
7664
7665     my $parser = sub {
7666         my $line = shift;
7667         if($line =~ m/\((\S+)\/100\%\)/){
7668             my $percent = $1;
7669             my $transferred = int($size * $percent / 100);
7670             my $total_h = render_bytes($size, 1);
7671             my $transferred_h = render_bytes($transferred, 1);
7672
7673             print "transferred $transferred_h of $total_h ($percent%)\n";
7674         }
7675
7676     };
7677
7678     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7679     my $err = $@;
7680     die "copy failed: $err" if $err;
7681 }
7682
7683 sub qemu_img_format {
7684     my ($scfg, $volname) = @_;
7685
7686     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7687         return $1;
7688     } else {
7689         return "raw";
7690     }
7691 }
7692
7693 sub qemu_drive_mirror {
7694     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7695
7696     $jobs = {} if !$jobs;
7697
7698     my $qemu_target;
7699     my $format;
7700     $jobs->{"drive-$drive"} = {};
7701
7702     if ($dst_volid =~ /^nbd:/) {
7703         $qemu_target = $dst_volid;
7704         $format = "nbd";
7705     } else {
7706         my $storecfg = PVE::Storage::config();
7707         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7708
7709         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7710
7711         $format = qemu_img_format($dst_scfg, $dst_volname);
7712
7713         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7714
7715         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7716     }
7717
7718     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7719     $opts->{format} = $format if $format;
7720
7721     if (defined($src_bitmap)) {
7722         $opts->{sync} = 'incremental';
7723         $opts->{bitmap} = $src_bitmap;
7724         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7725     }
7726
7727     if (defined($bwlimit)) {
7728         $opts->{speed} = $bwlimit * 1024;
7729         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7730     } else {
7731         print "drive mirror is starting for drive-$drive\n";
7732     }
7733
7734     # if a job already runs for this device we get an error, catch it for cleanup
7735     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7736     if (my $err = $@) {
7737         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7738         warn "$@\n" if $@;
7739         die "mirroring error: $err\n";
7740     }
7741
7742     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7743 }
7744
7745 # $completion can be either
7746 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7747 # 'cancel': wait until all jobs are ready, block-job-cancel them
7748 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7749 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7750 sub qemu_drive_mirror_monitor {
7751     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7752
7753     $completion //= 'complete';
7754     $op //= "mirror";
7755
7756     eval {
7757         my $err_complete = 0;
7758
7759         my $starttime = time ();
7760         while (1) {
7761             die "block job ('$op') timed out\n" if $err_complete > 300;
7762
7763             my $stats = mon_cmd($vmid, "query-block-jobs");
7764             my $ctime = time();
7765
7766             my $running_jobs = {};
7767             for my $stat (@$stats) {
7768                 next if $stat->{type} ne $op;
7769                 $running_jobs->{$stat->{device}} = $stat;
7770             }
7771
7772             my $readycounter = 0;
7773
7774             for my $job_id (sort keys %$jobs) {
7775                 my $job = $running_jobs->{$job_id};
7776
7777                 my $vanished = !defined($job);
7778                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7779                 if($complete || ($vanished && $completion eq 'auto')) {
7780                     print "$job_id: $op-job finished\n";
7781                     delete $jobs->{$job_id};
7782                     next;
7783                 }
7784
7785                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7786
7787                 my $busy = $job->{busy};
7788                 my $ready = $job->{ready};
7789                 if (my $total = $job->{len}) {
7790                     my $transferred = $job->{offset} || 0;
7791                     my $remaining = $total - $transferred;
7792                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7793
7794                     my $duration = $ctime - $starttime;
7795                     my $total_h = render_bytes($total, 1);
7796                     my $transferred_h = render_bytes($transferred, 1);
7797
7798                     my $status = sprintf(
7799                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7800                         render_duration($duration),
7801                     );
7802
7803                     if ($ready) {
7804                         if ($busy) {
7805                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7806                         } else {
7807                             $status .= ", ready";
7808                         }
7809                     }
7810                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7811                     $jobs->{$job_id}->{ready} = $ready;
7812                 }
7813
7814                 $readycounter++ if $job->{ready};
7815             }
7816
7817             last if scalar(keys %$jobs) == 0;
7818
7819             if ($readycounter == scalar(keys %$jobs)) {
7820                 print "all '$op' jobs are ready\n";
7821
7822                 # do the complete later (or has already been done)
7823                 last if $completion eq 'skip' || $completion eq 'auto';
7824
7825                 if ($vmiddst && $vmiddst != $vmid) {
7826                     my $agent_running = $qga && qga_check_running($vmid);
7827                     if ($agent_running) {
7828                         print "freeze filesystem\n";
7829                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7830                         warn $@ if $@;
7831                     } else {
7832                         print "suspend vm\n";
7833                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7834                         warn $@ if $@;
7835                     }
7836
7837                     # if we clone a disk for a new target vm, we don't switch the disk
7838                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7839
7840                     if ($agent_running) {
7841                         print "unfreeze filesystem\n";
7842                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7843                         warn $@ if $@;
7844                     } else {
7845                         print "resume vm\n";
7846                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7847                         warn $@ if $@;
7848                     }
7849
7850                     last;
7851                 } else {
7852
7853                     for my $job_id (sort keys %$jobs) {
7854                         # try to switch the disk if source and destination are on the same guest
7855                         print "$job_id: Completing block job_id...\n";
7856
7857                         my $op;
7858                         if ($completion eq 'complete') {
7859                             $op = 'block-job-complete';
7860                         } elsif ($completion eq 'cancel') {
7861                             $op = 'block-job-cancel';
7862                         } else {
7863                             die "invalid completion value: $completion\n";
7864                         }
7865                         eval { mon_cmd($vmid, $op, device => $job_id) };
7866                         if ($@ =~ m/cannot be completed/) {
7867                             print "$job_id: block job cannot be completed, trying again.\n";
7868                             $err_complete++;
7869                         }else {
7870                             print "$job_id: Completed successfully.\n";
7871                             $jobs->{$job_id}->{complete} = 1;
7872                         }
7873                     }
7874                 }
7875             }
7876             sleep 1;
7877         }
7878     };
7879     my $err = $@;
7880
7881     if ($err) {
7882         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7883         die "block job ($op) error: $err";
7884     }
7885 }
7886
7887 sub qemu_blockjobs_cancel {
7888     my ($vmid, $jobs) = @_;
7889
7890     foreach my $job (keys %$jobs) {
7891         print "$job: Cancelling block job\n";
7892         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7893         $jobs->{$job}->{cancel} = 1;
7894     }
7895
7896     while (1) {
7897         my $stats = mon_cmd($vmid, "query-block-jobs");
7898
7899         my $running_jobs = {};
7900         foreach my $stat (@$stats) {
7901             $running_jobs->{$stat->{device}} = $stat;
7902         }
7903
7904         foreach my $job (keys %$jobs) {
7905
7906             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7907                 print "$job: Done.\n";
7908                 delete $jobs->{$job};
7909             }
7910         }
7911
7912         last if scalar(keys %$jobs) == 0;
7913
7914         sleep 1;
7915     }
7916 }
7917
7918 sub clone_disk {
7919     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7920
7921     my ($vmid, $running) = $source->@{qw(vmid running)};
7922     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7923
7924     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7925     my ($storage, $format) = $dest->@{qw(storage format)};
7926
7927     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7928
7929     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7930         die "cloning from/to EFI disk requires EFI disk\n"
7931             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7932         die "cloning from/to TPM state requires TPM state\n"
7933             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7934
7935         # This would lead to two device nodes in QEMU pointing to the same backing image!
7936         die "cannot change drive name when cloning disk from/to the same VM\n"
7937             if $use_drive_mirror && $vmid == $newvmid;
7938     }
7939
7940     die "cannot move TPM state while VM is running\n"
7941         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7942
7943     my $newvolid;
7944
7945     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7946     print "$src_drivename " if $src_drivename;
7947     print "($drive->{file})\n";
7948
7949     if (!$full) {
7950         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7951         push @$newvollist, $newvolid;
7952     } else {
7953
7954         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7955         $storeid = $storage if $storage;
7956
7957         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7958
7959         my $name = undef;
7960         my $size = undef;
7961         if (drive_is_cloudinit($drive)) {
7962             $name = "vm-$newvmid-cloudinit";
7963             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7964             if ($scfg->{path}) {
7965                 $name .= ".$dst_format";
7966             }
7967             $snapname = undef;
7968             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7969         } elsif ($dst_drivename eq 'efidisk0') {
7970             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7971         } elsif ($dst_drivename eq 'tpmstate0') {
7972             $dst_format = 'raw';
7973             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7974         } else {
7975             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7976         }
7977         $newvolid = PVE::Storage::vdisk_alloc(
7978             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7979         );
7980         push @$newvollist, $newvolid;
7981
7982         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7983
7984         if (drive_is_cloudinit($drive)) {
7985             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7986             # if this is the case, we have to complete any block-jobs still there from
7987             # previous drive-mirrors
7988             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7989                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7990             }
7991             goto no_data_clone;
7992         }
7993
7994         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7995         if ($use_drive_mirror) {
7996             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7997                 $completion, $qga, $bwlimit);
7998         } else {
7999             # TODO: handle bwlimits
8000             if ($dst_drivename eq 'efidisk0') {
8001                 # the relevant data on the efidisk may be smaller than the source
8002                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8003                 # that is given by the OVMF_VARS.fd
8004                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8005                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8006
8007                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8008
8009                 # better for Ceph if block size is not too small, see bug #3324
8010                 my $bs = 1024*1024;
8011
8012                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8013
8014                 if ($src_format eq 'qcow2' && $snapname) {
8015                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8016                         if !min_version(kvm_user_version(), 6, 2);
8017                     push $cmd->@*, '-l', $snapname;
8018                 }
8019                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8020                 run_command($cmd);
8021             } else {
8022                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
8023             }
8024         }
8025     }
8026
8027 no_data_clone:
8028     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8029
8030     my $disk = dclone($drive);
8031     delete $disk->{format};
8032     $disk->{file} = $newvolid;
8033     $disk->{size} = $size if defined($size);
8034
8035     return $disk;
8036 }
8037
8038 sub get_running_qemu_version {
8039     my ($vmid) = @_;
8040     my $res = mon_cmd($vmid, "query-version");
8041     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8042 }
8043
8044 sub qemu_use_old_bios_files {
8045     my ($machine_type) = @_;
8046
8047     return if !$machine_type;
8048
8049     my $use_old_bios_files = undef;
8050
8051     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8052         $machine_type = $1;
8053         $use_old_bios_files = 1;
8054     } else {
8055         my $version = extract_version($machine_type, kvm_user_version());
8056         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8057         # load new efi bios files on migration. So this hack is required to allow
8058         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8059         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8060         $use_old_bios_files = !min_version($version, 2, 4);
8061     }
8062
8063     return ($use_old_bios_files, $machine_type);
8064 }
8065
8066 sub get_efivars_size {
8067     my ($conf, $efidisk) = @_;
8068
8069     my $arch = get_vm_arch($conf);
8070     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8071     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8072     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8073     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
8074     return -s $ovmf_vars;
8075 }
8076
8077 sub update_efidisk_size {
8078     my ($conf) = @_;
8079
8080     return if !defined($conf->{efidisk0});
8081
8082     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8083     $disk->{size} = get_efivars_size($conf);
8084     $conf->{efidisk0} = print_drive($disk);
8085
8086     return;
8087 }
8088
8089 sub update_tpmstate_size {
8090     my ($conf) = @_;
8091
8092     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8093     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8094     $conf->{tpmstate0} = print_drive($disk);
8095 }
8096
8097 sub create_efidisk($$$$$$$) {
8098     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8099
8100     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8101     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
8102
8103     my $vars_size_b = -s $ovmf_vars;
8104     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8105     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8106     PVE::Storage::activate_volumes($storecfg, [$volid]);
8107
8108     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8109     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8110
8111     return ($volid, $size/1024);
8112 }
8113
8114 sub vm_iothreads_list {
8115     my ($vmid) = @_;
8116
8117     my $res = mon_cmd($vmid, 'query-iothreads');
8118
8119     my $iothreads = {};
8120     foreach my $iothread (@$res) {
8121         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8122     }
8123
8124     return $iothreads;
8125 }
8126
8127 sub scsihw_infos {
8128     my ($conf, $drive) = @_;
8129
8130     my $maxdev = 0;
8131
8132     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8133         $maxdev = 7;
8134     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8135         $maxdev = 1;
8136     } else {
8137         $maxdev = 256;
8138     }
8139
8140     my $controller = int($drive->{index} / $maxdev);
8141     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8142         ? "virtioscsi"
8143         : "scsihw";
8144
8145     return ($maxdev, $controller, $controller_prefix);
8146 }
8147
8148 sub resolve_dst_disk_format {
8149         my ($storecfg, $storeid, $src_volname, $format) = @_;
8150         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8151
8152         if (!$format) {
8153             # if no target format is specified, use the source disk format as hint
8154             if ($src_volname) {
8155                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8156                 $format = qemu_img_format($scfg, $src_volname);
8157             } else {
8158                 return $defFormat;
8159             }
8160         }
8161
8162         # test if requested format is supported - else use default
8163         my $supported = grep { $_ eq $format } @$validFormats;
8164         $format = $defFormat if !$supported;
8165         return $format;
8166 }
8167
8168 # NOTE: if this logic changes, please update docs & possibly gui logic
8169 sub find_vmstate_storage {
8170     my ($conf, $storecfg) = @_;
8171
8172     # first, return storage from conf if set
8173     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8174
8175     my ($target, $shared, $local);
8176
8177     foreach_storage_used_by_vm($conf, sub {
8178         my ($sid) = @_;
8179         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8180         my $dst = $scfg->{shared} ? \$shared : \$local;
8181         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8182     });
8183
8184     # second, use shared storage where VM has at least one disk
8185     # third, use local storage where VM has at least one disk
8186     # fall back to local storage
8187     $target = $shared // $local // 'local';
8188
8189     return $target;
8190 }
8191
8192 sub generate_uuid {
8193     my ($uuid, $uuid_str);
8194     UUID::generate($uuid);
8195     UUID::unparse($uuid, $uuid_str);
8196     return $uuid_str;
8197 }
8198
8199 sub generate_smbios1_uuid {
8200     return "uuid=".generate_uuid();
8201 }
8202
8203 sub nbd_stop {
8204     my ($vmid) = @_;
8205
8206     mon_cmd($vmid, 'nbd-server-stop');
8207 }
8208
8209 sub create_reboot_request {
8210     my ($vmid) = @_;
8211     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8212         or die "failed to create reboot trigger file: $!\n";
8213     close($fh);
8214 }
8215
8216 sub clear_reboot_request {
8217     my ($vmid) = @_;
8218     my $path = "/run/qemu-server/$vmid.reboot";
8219     my $res = 0;
8220
8221     $res = unlink($path);
8222     die "could not remove reboot request for $vmid: $!"
8223         if !$res && $! != POSIX::ENOENT;
8224
8225     return $res;
8226 }
8227
8228 sub bootorder_from_legacy {
8229     my ($conf, $bootcfg) = @_;
8230
8231     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8232     my $bootindex_hash = {};
8233     my $i = 1;
8234     foreach my $o (split(//, $boot)) {
8235         $bootindex_hash->{$o} = $i*100;
8236         $i++;
8237     }
8238
8239     my $bootorder = {};
8240
8241     PVE::QemuConfig->foreach_volume($conf, sub {
8242         my ($ds, $drive) = @_;
8243
8244         if (drive_is_cdrom ($drive, 1)) {
8245             if ($bootindex_hash->{d}) {
8246                 $bootorder->{$ds} = $bootindex_hash->{d};
8247                 $bootindex_hash->{d} += 1;
8248             }
8249         } elsif ($bootindex_hash->{c}) {
8250             $bootorder->{$ds} = $bootindex_hash->{c}
8251                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8252             $bootindex_hash->{c} += 1;
8253         }
8254     });
8255
8256     if ($bootindex_hash->{n}) {
8257         for (my $i = 0; $i < $MAX_NETS; $i++) {
8258             my $netname = "net$i";
8259             next if !$conf->{$netname};
8260             $bootorder->{$netname} = $bootindex_hash->{n};
8261             $bootindex_hash->{n} += 1;
8262         }
8263     }
8264
8265     return $bootorder;
8266 }
8267
8268 # Generate default device list for 'boot: order=' property. Matches legacy
8269 # default boot order, but with explicit device names. This is important, since
8270 # the fallback for when neither 'order' nor the old format is specified relies
8271 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8272 sub get_default_bootdevices {
8273     my ($conf) = @_;
8274
8275     my @ret = ();
8276
8277     # harddisk
8278     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8279     push @ret, $first if $first;
8280
8281     # cdrom
8282     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8283     push @ret, $first if $first;
8284
8285     # network
8286     for (my $i = 0; $i < $MAX_NETS; $i++) {
8287         my $netname = "net$i";
8288         next if !$conf->{$netname};
8289         push @ret, $netname;
8290         last;
8291     }
8292
8293     return \@ret;
8294 }
8295
8296 sub device_bootorder {
8297     my ($conf) = @_;
8298
8299     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8300
8301     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8302
8303     my $bootorder = {};
8304     if (!defined($boot) || $boot->{legacy}) {
8305         $bootorder = bootorder_from_legacy($conf, $boot);
8306     } elsif ($boot->{order}) {
8307         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8308         for my $dev (PVE::Tools::split_list($boot->{order})) {
8309             $bootorder->{$dev} = $i++;
8310         }
8311     }
8312
8313     return $bootorder;
8314 }
8315
8316 sub register_qmeventd_handle {
8317     my ($vmid) = @_;
8318
8319     my $fh;
8320     my $peer = "/var/run/qmeventd.sock";
8321     my $count = 0;
8322
8323     for (;;) {
8324         $count++;
8325         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8326         last if $fh;
8327         if ($! != EINTR && $! != EAGAIN) {
8328             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8329         }
8330         if ($count > 4) {
8331             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8332               . "after $count retries\n";
8333         }
8334         usleep(25000);
8335     }
8336
8337     # send handshake to mark VM as backing up
8338     print $fh to_json({vzdump => {vmid => "$vmid"}});
8339
8340     # return handle to be closed later when inhibit is no longer required
8341     return $fh;
8342 }
8343
8344 # bash completion helper
8345
8346 sub complete_backup_archives {
8347     my ($cmdname, $pname, $cvalue) = @_;
8348
8349     my $cfg = PVE::Storage::config();
8350
8351     my $storeid;
8352
8353     if ($cvalue =~ m/^([^:]+):/) {
8354         $storeid = $1;
8355     }
8356
8357     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8358
8359     my $res = [];
8360     foreach my $id (keys %$data) {
8361         foreach my $item (@{$data->{$id}}) {
8362             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8363             push @$res, $item->{volid} if defined($item->{volid});
8364         }
8365     }
8366
8367     return $res;
8368 }
8369
8370 my $complete_vmid_full = sub {
8371     my ($running) = @_;
8372
8373     my $idlist = vmstatus();
8374
8375     my $res = [];
8376
8377     foreach my $id (keys %$idlist) {
8378         my $d = $idlist->{$id};
8379         if (defined($running)) {
8380             next if $d->{template};
8381             next if $running && $d->{status} ne 'running';
8382             next if !$running && $d->{status} eq 'running';
8383         }
8384         push @$res, $id;
8385
8386     }
8387     return $res;
8388 };
8389
8390 sub complete_vmid {
8391     return &$complete_vmid_full();
8392 }
8393
8394 sub complete_vmid_stopped {
8395     return &$complete_vmid_full(0);
8396 }
8397
8398 sub complete_vmid_running {
8399     return &$complete_vmid_full(1);
8400 }
8401
8402 sub complete_storage {
8403
8404     my $cfg = PVE::Storage::config();
8405     my $ids = $cfg->{ids};
8406
8407     my $res = [];
8408     foreach my $sid (keys %$ids) {
8409         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8410         next if !$ids->{$sid}->{content}->{images};
8411         push @$res, $sid;
8412     }
8413
8414     return $res;
8415 }
8416
8417 sub complete_migration_storage {
8418     my ($cmd, $param, $current_value, $all_args) = @_;
8419
8420     my $targetnode = @$all_args[1];
8421
8422     my $cfg = PVE::Storage::config();
8423     my $ids = $cfg->{ids};
8424
8425     my $res = [];
8426     foreach my $sid (keys %$ids) {
8427         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8428         next if !$ids->{$sid}->{content}->{images};
8429         push @$res, $sid;
8430     }
8431
8432     return $res;
8433 }
8434
8435 sub vm_is_paused {
8436     my ($vmid) = @_;
8437     my $qmpstatus = eval {
8438         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8439         mon_cmd($vmid, "query-status");
8440     };
8441     warn "$@\n" if $@;
8442     return $qmpstatus && $qmpstatus->{status} eq "paused";
8443 }
8444
8445 sub check_volume_storage_type {
8446     my ($storecfg, $vol) = @_;
8447
8448     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8449     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8450     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8451
8452     die "storage '$storeid' does not support content-type '$vtype'\n"
8453         if !$scfg->{content}->{$vtype};
8454
8455     return 1;
8456 }
8457
8458 sub add_nets_bridge_fdb {
8459     my ($conf, $vmid) = @_;
8460
8461     for my $opt (keys %$conf) {
8462         next if $opt !~ m/^net(\d+)$/;
8463         my $iface = "tap${vmid}i$1";
8464         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8465         my $net = parse_net($conf->{$opt}, 1) or next;
8466
8467         my $mac = $net->{macaddr};
8468         if (!$mac) {
8469             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8470                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8471             next;
8472         }
8473
8474         my $bridge = $net->{bridge};
8475         if ($have_sdn) {
8476             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8477         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8478             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8479         }
8480     }
8481 }
8482
8483 sub del_nets_bridge_fdb {
8484     my ($conf, $vmid) = @_;
8485
8486     for my $opt (keys %$conf) {
8487         next if $opt !~ m/^net(\d+)$/;
8488         my $iface = "tap${vmid}i$1";
8489
8490         my $net = parse_net($conf->{$opt}) or next;
8491         my $mac = $net->{macaddr} or next;
8492
8493         my $bridge = $net->{bridge};
8494         if ($have_sdn) {
8495             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8496         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8497             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8498         }
8499     }
8500 }
8501
8502 1;