PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB qw(parse_usb_device);
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 244             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 245             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 246             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 247             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 248             ." a hardware RNG from the host.",
 249     },
 250     max_bytes => {
 251         type => 'integer',
 252         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 253             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 254             ." `0` to disable limiting (potentially dangerous!).",
 255         optional => 1,
 256
 257         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 258         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 259         # reading from /dev/urandom
 260         default => 1024,
 261     },
 262     period => {
 263         type => 'integer',
 264         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 265             ." the guest to retrieve another 'max_bytes' of entropy.",
 266         optional => 1,
 267         default => 1000,
 268     },
 269 };
 270
 271 my $meta_info_fmt = {
 272     'ctime' => {
 273         type => 'integer',
 274         description => "The guest creation timestamp as UNIX epoch time",
 275         minimum => 0,
 276         optional => 1,
 277     },
 278     'creation-qemu' => {
 279         type => 'string',
 280         description => "The QEMU (machine) version from the time this VM was created.",
 281         pattern => '\d+(\.\d+)+',
 282         optional => 1,
 283     },
 284 };
 285
 286 my $confdesc = {
 287     onboot => {
 288         optional => 1,
 289         type => 'boolean',
 290         description => "Specifies whether a VM will be started during system bootup.",
 291         default => 0,
 292     },
 293     autostart => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Automatic restart after crash (currently ignored).",
 297         default => 0,
 298     },
 299     hotplug => {
 300         optional => 1,
 301         type => 'string', format => 'pve-hotplug-features',
 302         description => "Selectively enable hotplug features. This is a comma separated list of"
 303             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 304             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 305             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 306             ." windows > 7.",
 307         default => 'network,disk,usb',
 308     },
 309     reboot => {
 310         optional => 1,
 311         type => 'boolean',
 312         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 313         default => 1,
 314     },
 315     lock => {
 316         optional => 1,
 317         type => 'string',
 318         description => "Lock/unlock the VM.",
 319         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 320     },
 321     cpulimit => {
 322         optional => 1,
 323         type => 'number',
 324         description => "Limit of CPU usage.",
 325         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 326             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 327         minimum => 0,
 328         maximum => 128,
 329         default => 0,
 330     },
 331     cpuunits => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 335         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 336             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 337             ." weights of all the other running VMs.",
 338         minimum => 1,
 339         maximum => 262144,
 340         default => 'cgroup v1: 1024, cgroup v2: 100',
 341     },
 342     memory => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 346             ." you use the balloon device.",
 347         minimum => 16,
 348         default => 512,
 349     },
 350     balloon => {
 351         optional => 1,
 352         type => 'integer',
 353         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 354         minimum => 0,
 355     },
 356     shares => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 360             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 361             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 362         minimum => 0,
 363         maximum => 50000,
 364         default => 1000,
 365     },
 366     keyboard => {
 367         optional => 1,
 368         type => 'string',
 369         description => "Keyboard layout for VNC server. This option is generally not required and"
 370         ." is often better handled from within the guest OS.",
 371         enum => PVE::Tools::kvmkeymaplist(),
 372         default => undef,
 373     },
 374     name => {
 375         optional => 1,
 376         type => 'string', format => 'dns-name',
 377         description => "Set a name for the VM. Only used on the configuration web interface.",
 378     },
 379     scsihw => {
 380         optional => 1,
 381         type => 'string',
 382         description => "SCSI controller model",
 383         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 384         default => 'lsi',
 385     },
 386     description => {
 387         optional => 1,
 388         type => 'string',
 389         description => "Description for the VM. Shown in the web-interface VM's summary."
 390             ." This is saved as comment inside the configuration file.",
 391         maxLength => 1024 * 8,
 392     },
 393     ostype => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 397         description => "Specify guest operating system.",
 398         verbose_description => <<EODESC,
 399 Specify guest operating system. This is used to enable special
 400 optimization/features for specific operating systems:
 401
 402 [horizontal]
 403 other;; unspecified OS
 404 wxp;; Microsoft Windows XP
 405 w2k;; Microsoft Windows 2000
 406 w2k3;; Microsoft Windows 2003
 407 w2k8;; Microsoft Windows 2008
 408 wvista;; Microsoft Windows Vista
 409 win7;; Microsoft Windows 7
 410 win8;; Microsoft Windows 8/2012/2012r2
 411 win10;; Microsoft Windows 10/2016/2019
 412 win11;; Microsoft Windows 11/2022
 413 l24;; Linux 2.4 Kernel
 414 l26;; Linux 2.6 - 5.X Kernel
 415 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 416 EODESC
 417     },
 418     boot => {
 419         optional => 1,
 420         type => 'string', format => 'pve-qm-boot',
 421         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 422             ." key or 'legacy=' is deprecated.",
 423     },
 424     bootdisk => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-bootdisk',
 427         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 428         pattern => '(ide|sata|scsi|virtio)\d+',
 429     },
 430     smp => {
 431         optional => 1,
 432         type => 'integer',
 433         description => "The number of CPUs. Please use option -sockets instead.",
 434         minimum => 1,
 435         default => 1,
 436     },
 437     sockets => {
 438         optional => 1,
 439         type => 'integer',
 440         description => "The number of CPU sockets.",
 441         minimum => 1,
 442         default => 1,
 443     },
 444     cores => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "The number of cores per socket.",
 448         minimum => 1,
 449         default => 1,
 450     },
 451     numa => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable NUMA.",
 455         default => 0,
 456     },
 457     hugepages => {
 458         optional => 1,
 459         type => 'string',
 460         description => "Enable/disable hugepages memory.",
 461         enum => [qw(any 2 1024)],
 462     },
 463     keephugepages => {
 464         optional => 1,
 465         type => 'boolean',
 466         default => 0,
 467         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 468             ." after VM shutdown and can be used for subsequent starts.",
 469     },
 470     vcpus => {
 471         optional => 1,
 472         type => 'integer',
 473         description => "Number of hotplugged vcpus.",
 474         minimum => 1,
 475         default => 0,
 476     },
 477     acpi => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Enable/disable ACPI.",
 481         default => 1,
 482     },
 483     agent => {
 484         optional => 1,
 485         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 486         type => 'string',
 487         format => $agent_fmt,
 488     },
 489     kvm => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Enable/disable KVM hardware virtualization.",
 493         default => 1,
 494     },
 495     tdf => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable time drift fix.",
 499         default => 0,
 500     },
 501     localtime => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 505             ." the `ostype` indicates a Microsoft Windows OS.",
 506     },
 507     freeze => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 511     },
 512     vga => {
 513         optional => 1,
 514         type => 'string', format => $vga_fmt,
 515         description => "Configure the VGA hardware.",
 516         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 517             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 518             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 519             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 520             ." display server. For win* OS you can select how many independent displays you want,"
 521             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 522             ." using a serial device as terminal.",
 523     },
 524     watchdog => {
 525         optional => 1,
 526         type => 'string', format => 'pve-qm-watchdog',
 527         description => "Create a virtual hardware watchdog device.",
 528         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 529             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 530             ." else the watchdog will reset the guest (or execute the respective action specified)",
 531     },
 532     startdate => {
 533         optional => 1,
 534         type => 'string',
 535         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 536         description => "Set the initial date of the real time clock. Valid format for date are:"
 537             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 538         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 539         default => 'now',
 540     },
 541     startup =>  get_standard_option('pve-startup-order'),
 542     template => {
 543         optional => 1,
 544         type => 'boolean',
 545         description => "Enable/disable Template.",
 546         default => 0,
 547     },
 548     args => {
 549         optional => 1,
 550         type => 'string',
 551         description => "Arbitrary arguments passed to kvm.",
 552         verbose_description => <<EODESCR,
 553 Arbitrary arguments passed to kvm, for example:
 554
 555 args: -no-reboot -no-hpet
 556
 557 NOTE: this option is for experts only.
 558 EODESCR
 559     },
 560     tablet => {
 561         optional => 1,
 562         type => 'boolean',
 563         default => 1,
 564         description => "Enable/disable the USB tablet device.",
 565         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 566             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 567             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 568             ." may consider disabling this to save some context switches. This is turned off by"
 569             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 570     },
 571     migrate_speed => {
 572         optional => 1,
 573         type => 'integer',
 574         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 575         minimum => 0,
 576         default => 0,
 577     },
 578     migrate_downtime => {
 579         optional => 1,
 580         type => 'number',
 581         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 582         minimum => 0,
 583         default => 0.1,
 584     },
 585     cdrom => {
 586         optional => 1,
 587         type => 'string', format => 'pve-qm-ide',
 588         typetext => '<volume>',
 589         description => "This is an alias for option -ide2",
 590     },
 591     cpu => {
 592         optional => 1,
 593         description => "Emulated CPU type.",
 594         type => 'string',
 595         format => 'pve-vm-cpu-conf',
 596     },
 597     parent => get_standard_option('pve-snapshot-name', {
 598         optional => 1,
 599         description => "Parent snapshot name. This is used internally, and should not be modified.",
 600     }),
 601     snaptime => {
 602         optional => 1,
 603         description => "Timestamp for snapshots.",
 604         type => 'integer',
 605         minimum => 0,
 606     },
 607     vmstate => {
 608         optional => 1,
 609         type => 'string', format => 'pve-volume-id',
 610         description => "Reference to a volume which stores the VM state. This is used internally"
 611             ." for snapshots.",
 612     },
 613     vmstatestorage => get_standard_option('pve-storage-id', {
 614         description => "Default storage for VM state volumes/files.",
 615         optional => 1,
 616     }),
 617     runningmachine => get_standard_option('pve-qemu-machine', {
 618         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 619             ." for snapshots.",
 620     }),
 621     runningcpu => {
 622         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 623             ." internally for snapshots.",
 624         optional => 1,
 625         type => 'string',
 626         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 627         format_description => 'QEMU -cpu parameter'
 628     },
 629     machine => get_standard_option('pve-qemu-machine'),
 630     arch => {
 631         description => "Virtual processor architecture. Defaults to the host.",
 632         optional => 1,
 633         type => 'string',
 634         enum => [qw(x86_64 aarch64)],
 635     },
 636     smbios1 => {
 637         description => "Specify SMBIOS type 1 fields.",
 638         type => 'string', format => 'pve-qm-smbios1',
 639         maxLength => 512,
 640         optional => 1,
 641     },
 642     protection => {
 643         optional => 1,
 644         type => 'boolean',
 645         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 646             ." remove disk operations.",
 647         default => 0,
 648     },
 649     bios => {
 650         optional => 1,
 651         type => 'string',
 652         enum => [ qw(seabios ovmf) ],
 653         description => "Select BIOS implementation.",
 654         default => 'seabios',
 655     },
 656     vmgenid => {
 657         type => 'string',
 658         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 659         format_description => 'UUID',
 660         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 661             ." to disable explicitly.",
 662         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 663             ." value identifier to the guest OS. This allows to notify the guest operating system"
 664             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 665             ." execution or creation from a template). The guest operating system notices the"
 666             ." change, and is then able to react as appropriate by marking its copies of"
 667             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 668             ."Note that auto-creation only works when done through API/CLI create or update methods"
 669             .", but not when manually editing the config file.",
 670         default => "1 (autogenerated)",
 671         optional => 1,
 672     },
 673     hookscript => {
 674         type => 'string',
 675         format => 'pve-volume-id',
 676         optional => 1,
 677         description => "Script that will be executed during various steps in the vms lifetime.",
 678     },
 679     ivshmem => {
 680         type => 'string',
 681         format => $ivshmem_fmt,
 682         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 683             ." the host.",
 684         optional => 1,
 685     },
 686     audio0 => {
 687         type => 'string',
 688         format => $audio_fmt,
 689         description => "Configure a audio device, useful in combination with QXL/Spice.",
 690         optional => 1
 691     },
 692     spice_enhancements => {
 693         type => 'string',
 694         format => $spice_enhancements_fmt,
 695         description => "Configure additional enhancements for SPICE.",
 696         optional => 1
 697     },
 698     tags => {
 699         type => 'string', format => 'pve-tag-list',
 700         description => 'Tags of the VM. This is only meta information.',
 701         optional => 1,
 702     },
 703     rng0 => {
 704         type => 'string',
 705         format => $rng_fmt,
 706         description => "Configure a VirtIO-based Random Number Generator.",
 707         optional => 1,
 708     },
 709     meta => {
 710         type => 'string',
 711         format => $meta_info_fmt,
 712         description => "Some (read-only) meta-information about this guest.",
 713         optional => 1,
 714     },
 715     affinity => {
 716         type => 'string', format => 'pve-cpuset',
 717         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 718         optional => 1,
 719     },
 720 };
 721
 722 my $cicustom_fmt = {
 723     meta => {
 724         type => 'string',
 725         optional => 1,
 726         description => 'Specify a custom file containing all meta data passed to the VM via"
 727             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731     network => {
 732         type => 'string',
 733         optional => 1,
 734         description => 'Specify a custom file containing all network data passed to the VM via'
 735             .' cloud-init.',
 736         format => 'pve-volume-id',
 737         format_description => 'volume',
 738     },
 739     user => {
 740         type => 'string',
 741         optional => 1,
 742         description => 'Specify a custom file containing all user data passed to the VM via'
 743             .' cloud-init.',
 744         format => 'pve-volume-id',
 745         format_description => 'volume',
 746     },
 747     vendor => {
 748     type => 'string',
 749     optional => 1,
 750     description => 'Specify a custom file containing all vendor data passed to the VM via'
 751      .' cloud-init.',
 752     format => 'pve-volume-id',
 753     format_description => 'volume',
 754     },
 755 };
 756 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 757
 758 my $confdesc_cloudinit = {
 759     citype => {
 760         optional => 1,
 761         type => 'string',
 762         description => 'Specifies the cloud-init configuration format. The default depends on the'
 763             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 764             .' and `configdrive2` for windows.',
 765         enum => ['configdrive2', 'nocloud', 'opennebula'],
 766     },
 767     ciuser => {
 768         optional => 1,
 769         type => 'string',
 770         description => "cloud-init: User name to change ssh keys and password for instead of the"
 771             ." image's configured default user.",
 772     },
 773     cipassword => {
 774         optional => 1,
 775         type => 'string',
 776         description => 'cloud-init: Password to assign the user. Using this is generally not'
 777             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 778             .' support hashed passwords.',
 779     },
 780     cicustom => {
 781         optional => 1,
 782         type => 'string',
 783         description => 'cloud-init: Specify custom files to replace the automatically generated'
 784             .' ones at start.',
 785         format => 'pve-qm-cicustom',
 786     },
 787     searchdomain => {
 788         optional => 1,
 789         type => 'string',
 790         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 791             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 792             .' are set.',
 793     },
 794     nameserver => {
 795         optional => 1,
 796         type => 'string', format => 'address-list',
 797         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 798             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 799             .' are set.',
 800     },
 801     sshkeys => {
 802         optional => 1,
 803         type => 'string',
 804         format => 'urlencoded',
 805         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 806     },
 807 };
 808
 809 # what about other qemu settings ?
 810 #cpu => 'string',
 811 #machine => 'string',
 812 #fda => 'file',
 813 #fdb => 'file',
 814 #mtdblock => 'file',
 815 #sd => 'file',
 816 #pflash => 'file',
 817 #snapshot => 'bool',
 818 #bootp => 'file',
 819 ##tftp => 'dir',
 820 ##smb => 'dir',
 821 #kernel => 'file',
 822 #append => 'string',
 823 #initrd => 'file',
 824 ##soundhw => 'string',
 825
 826 while (my ($k, $v) = each %$confdesc) {
 827     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 828 }
 829
 830 my $MAX_USB_DEVICES = 14;
 831 my $MAX_NETS = 32;
 832 my $MAX_SERIAL_PORTS = 4;
 833 my $MAX_PARALLEL_PORTS = 3;
 834 my $MAX_NUMA = 8;
 835
 836 my $numa_fmt = {
 837     cpus => {
 838         type => "string",
 839         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 840         description => "CPUs accessing this NUMA node.",
 841         format_description => "id[-id];...",
 842     },
 843     memory => {
 844         type => "number",
 845         description => "Amount of memory this NUMA node provides.",
 846         optional => 1,
 847     },
 848     hostnodes => {
 849         type => "string",
 850         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 851         description => "Host NUMA nodes to use.",
 852         format_description => "id[-id];...",
 853         optional => 1,
 854     },
 855     policy => {
 856         type => 'string',
 857         enum => [qw(preferred bind interleave)],
 858         description => "NUMA allocation policy.",
 859         optional => 1,
 860     },
 861 };
 862 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 863 my $numadesc = {
 864     optional => 1,
 865     type => 'string', format => $numa_fmt,
 866     description => "NUMA topology.",
 867 };
 868 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 869
 870 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 871     $confdesc->{"numa$i"} = $numadesc;
 872 }
 873
 874 my $nic_model_list = [
 875     'e1000',
 876     'e1000-82540em',
 877     'e1000-82544gc',
 878     'e1000-82545em',
 879     'e1000e',
 880     'i82551',
 881     'i82557b',
 882     'i82559er',
 883     'ne2k_isa',
 884     'ne2k_pci',
 885     'pcnet',
 886     'rtl8139',
 887     'virtio',
 888     'vmxnet3',
 889 ];
 890 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 891
 892 my $net_fmt_bridge_descr = <<__EOD__;
 893 Bridge to attach the network device to. The Proxmox VE standard bridge
 894 is called 'vmbr0'.
 895
 896 If you do not specify a bridge, we create a kvm user (NATed) network
 897 device, which provides DHCP and DNS services. The following addresses
 898 are used:
 899
 900  10.0.2.2   Gateway
 901  10.0.2.3   DNS Server
 902  10.0.2.4   SMB Server
 903
 904 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 905 __EOD__
 906
 907 my $net_fmt = {
 908     macaddr  => get_standard_option('mac-addr', {
 909         description => "MAC address. That address must be unique withing your network. This is"
 910             ." automatically generated if not specified.",
 911     }),
 912     model => {
 913         type => 'string',
 914         description => "Network Card Model. The 'virtio' model provides the best performance with"
 915             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 916             ." best to use 'e1000'.",
 917         enum => $nic_model_list,
 918         default_key => 1,
 919     },
 920     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 921     bridge => get_standard_option('pve-bridge-id', {
 922         description => $net_fmt_bridge_descr,
 923         optional => 1,
 924     }),
 925     queues => {
 926         type => 'integer',
 927         minimum => 0, maximum => 64,
 928         description => 'Number of packet queues to be used on the device.',
 929         optional => 1,
 930     },
 931     rate => {
 932         type => 'number',
 933         minimum => 0,
 934         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 935         optional => 1,
 936     },
 937     tag => {
 938         type => 'integer',
 939         minimum => 1, maximum => 4094,
 940         description => 'VLAN tag to apply to packets on this interface.',
 941         optional => 1,
 942     },
 943     trunks => {
 944         type => 'string',
 945         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 946         description => 'VLAN trunks to pass through this interface.',
 947         format_description => 'vlanid[;vlanid...]',
 948         optional => 1,
 949     },
 950     firewall => {
 951         type => 'boolean',
 952         description => 'Whether this interface should be protected by the firewall.',
 953         optional => 1,
 954     },
 955     link_down => {
 956         type => 'boolean',
 957         description => 'Whether this interface should be disconnected (like pulling the plug).',
 958         optional => 1,
 959     },
 960     mtu => {
 961         type => 'integer',
 962         minimum => 1, maximum => 65520,
 963         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 964         optional => 1,
 965     },
 966 };
 967
 968 my $netdesc = {
 969     optional => 1,
 970     type => 'string', format => $net_fmt,
 971     description => "Specify network devices.",
 972 };
 973
 974 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 975
 976 my $ipconfig_fmt = {
 977     ip => {
 978         type => 'string',
 979         format => 'pve-ipv4-config',
 980         format_description => 'IPv4Format/CIDR',
 981         description => 'IPv4 address in CIDR format.',
 982         optional => 1,
 983         default => 'dhcp',
 984     },
 985     gw => {
 986         type => 'string',
 987         format => 'ipv4',
 988         format_description => 'GatewayIPv4',
 989         description => 'Default gateway for IPv4 traffic.',
 990         optional => 1,
 991         requires => 'ip',
 992     },
 993     ip6 => {
 994         type => 'string',
 995         format => 'pve-ipv6-config',
 996         format_description => 'IPv6Format/CIDR',
 997         description => 'IPv6 address in CIDR format.',
 998         optional => 1,
 999         default => 'dhcp',
1000     },
1001     gw6 => {
1002         type => 'string',
1003         format => 'ipv6',
1004         format_description => 'GatewayIPv6',
1005         description => 'Default gateway for IPv6 traffic.',
1006         optional => 1,
1007         requires => 'ip6',
1008     },
1009 };
1010 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1011 my $ipconfigdesc = {
1012     optional => 1,
1013     type => 'string', format => 'pve-qm-ipconfig',
1014     description => <<'EODESCR',
1015 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1016
1017 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1018
1019 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1020 gateway should be provided.
1021 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1022 cloud-init 19.4 or newer.
1023
1024 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1025 dhcp on IPv4.
1026 EODESCR
1027 };
1028 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1029
1030 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1031     $confdesc->{"net$i"} = $netdesc;
1032     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1033 }
1034
1035 foreach my $key (keys %$confdesc_cloudinit) {
1036     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1037 }
1038
1039 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1040 sub pve_verify_cpuset {
1041     my ($set_text, $noerr) = @_;
1042
1043     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1044
1045     if ($@) {
1046         return if $noerr;
1047         die "unable to parse cpuset option\n";
1048     }
1049
1050     return PVE::CpuSet->new($members)->short_string();
1051 }
1052
1053 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1054 sub verify_volume_id_or_qm_path {
1055     my ($volid, $noerr) = @_;
1056
1057     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1058
1059     return verify_volume_id_or_absolute_path($volid, $noerr);
1060 }
1061
1062 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1063 sub verify_volume_id_or_absolute_path {
1064     my ($volid, $noerr) = @_;
1065
1066     return $volid if $volid =~ m|^/|;
1067
1068     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1069     if ($@) {
1070         return if $noerr;
1071         die $@;
1072     }
1073     return $volid;
1074 }
1075
1076 my $usb_fmt = {
1077     host => {
1078         default_key => 1,
1079         type => 'string', format => 'pve-qm-usb-device',
1080         format_description => 'HOSTUSBDEVICE|spice',
1081         description => <<EODESCR,
1082 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1083
1084  'bus-port(.port)*' (decimal numbers) or
1085  'vendor_id:product_id' (hexadeciaml numbers) or
1086  'spice'
1087
1088 You can use the 'lsusb -t' command to list existing usb devices.
1089
1090 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1091 machines - use with special care.
1092
1093 The value 'spice' can be used to add a usb redirection devices for spice.
1094 EODESCR
1095     },
1096     usb3 => {
1097         optional => 1,
1098         type => 'boolean',
1099         description => "Specifies whether if given host option is a USB3 device or port."
1100             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1101             ." is irrelevant (all devices are plugged into a xhci controller).",
1102         default => 0,
1103     },
1104 };
1105
1106 my $usbdesc = {
1107     optional => 1,
1108     type => 'string', format => $usb_fmt,
1109     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1110         ." l26 or windows > 7, n can be up to 14).",
1111 };
1112 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1113
1114 my $serialdesc = {
1115         optional => 1,
1116         type => 'string',
1117         pattern => '(/dev/.+|socket)',
1118         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1119         verbose_description =>  <<EODESCR,
1120 Create a serial device inside the VM (n is 0 to 3), and pass through a
1121 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1122 host side (use 'qm terminal' to open a terminal connection).
1123
1124 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1125 use with special care.
1126
1127 CAUTION: Experimental! User reported problems with this option.
1128 EODESCR
1129 };
1130
1131 my $paralleldesc= {
1132         optional => 1,
1133         type => 'string',
1134         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1135         description =>  "Map host parallel devices (n is 0 to 2).",
1136         verbose_description =>  <<EODESCR,
1137 Map host parallel devices (n is 0 to 2).
1138
1139 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1140 machines - use with special care.
1141
1142 CAUTION: Experimental! User reported problems with this option.
1143 EODESCR
1144 };
1145
1146 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1147     $confdesc->{"parallel$i"} = $paralleldesc;
1148 }
1149
1150 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1151     $confdesc->{"serial$i"} = $serialdesc;
1152 }
1153
1154 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1155     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1156 }
1157
1158 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1159     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1160 }
1161
1162 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1163     $confdesc->{"usb$i"} = $usbdesc;
1164 }
1165
1166 my $boot_fmt = {
1167     legacy => {
1168         optional => 1,
1169         default_key => 1,
1170         type => 'string',
1171         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1172                      . " Deprecated, use 'order=' instead.",
1173         pattern => '[acdn]{1,4}',
1174         format_description => "[acdn]{1,4}",
1175
1176         # note: this is also the fallback if boot: is not given at all
1177         default => 'cdn',
1178     },
1179     order => {
1180         optional => 1,
1181         type => 'string',
1182         format => 'pve-qm-bootdev-list',
1183         format_description => "device[;device...]",
1184         description => <<EODESC,
1185 The guest will attempt to boot from devices in the order they appear here.
1186
1187 Disks, optical drives and passed-through storage USB devices will be directly
1188 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1189 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1190
1191 Note that only devices in this list will be marked as bootable and thus loaded
1192 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1193 (e.g. software-raid), you need to specify all of them here.
1194
1195 Overrides the deprecated 'legacy=[acdn]*' value when given.
1196 EODESC
1197     },
1198 };
1199 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1200
1201 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1202 sub verify_bootdev {
1203     my ($dev, $noerr) = @_;
1204
1205     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1206     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1207
1208     my $check = sub {
1209         my ($base) = @_;
1210         return 0 if $dev !~ m/^$base\d+$/;
1211         return 0 if !$confdesc->{$dev};
1212         return 1;
1213     };
1214
1215     return $dev if $check->("net");
1216     return $dev if $check->("usb");
1217     return $dev if $check->("hostpci");
1218
1219     return if $noerr;
1220     die "invalid boot device '$dev'\n";
1221 }
1222
1223 sub print_bootorder {
1224     my ($devs) = @_;
1225     return "" if !@$devs;
1226     my $data = { order => join(';', @$devs) };
1227     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1228 }
1229
1230 my $kvm_api_version = 0;
1231
1232 sub kvm_version {
1233     return $kvm_api_version if $kvm_api_version;
1234
1235     open my $fh, '<', '/dev/kvm' or return;
1236
1237     # 0xae00 => KVM_GET_API_VERSION
1238     $kvm_api_version = ioctl($fh, 0xae00, 0);
1239     close($fh);
1240
1241     return $kvm_api_version;
1242 }
1243
1244 my $kvm_user_version = {};
1245 my $kvm_mtime = {};
1246
1247 sub kvm_user_version {
1248     my ($binary) = @_;
1249
1250     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1251     my $st = stat($binary);
1252
1253     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1254     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1255         $cachedmtime == $st->mtime;
1256
1257     $kvm_user_version->{$binary} = 'unknown';
1258     $kvm_mtime->{$binary} = $st->mtime;
1259
1260     my $code = sub {
1261         my $line = shift;
1262         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1263             $kvm_user_version->{$binary} = $2;
1264         }
1265     };
1266
1267     eval { run_command([$binary, '--version'], outfunc => $code); };
1268     warn $@ if $@;
1269
1270     return $kvm_user_version->{$binary};
1271
1272 }
1273 my sub extract_version {
1274     my ($machine_type, $version) = @_;
1275     $version = kvm_user_version() if !defined($version);
1276     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1277 }
1278
1279 sub kernel_has_vhost_net {
1280     return -c '/dev/vhost-net';
1281 }
1282
1283 sub option_exists {
1284     my $key = shift;
1285     return defined($confdesc->{$key});
1286 }
1287
1288 my $cdrom_path;
1289 sub get_cdrom_path {
1290
1291     return $cdrom_path if defined($cdrom_path);
1292
1293     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1294
1295     if (!defined($cdrom_path)) {
1296         log_warn("no physical CD-ROM available, ignoring");
1297         $cdrom_path = '';
1298     }
1299
1300     return $cdrom_path;
1301 }
1302
1303 sub get_iso_path {
1304     my ($storecfg, $vmid, $cdrom) = @_;
1305
1306     if ($cdrom eq 'cdrom') {
1307         return get_cdrom_path();
1308     } elsif ($cdrom eq 'none') {
1309         return '';
1310     } elsif ($cdrom =~ m|^/|) {
1311         return $cdrom;
1312     } else {
1313         return PVE::Storage::path($storecfg, $cdrom);
1314     }
1315 }
1316
1317 # try to convert old style file names to volume IDs
1318 sub filename_to_volume_id {
1319     my ($vmid, $file, $media) = @_;
1320
1321      if (!($file eq 'none' || $file eq 'cdrom' ||
1322           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1323
1324         return if $file =~ m|/|;
1325
1326         if ($media && $media eq 'cdrom') {
1327             $file = "local:iso/$file";
1328         } else {
1329             $file = "local:$vmid/$file";
1330         }
1331     }
1332
1333     return $file;
1334 }
1335
1336 sub verify_media_type {
1337     my ($opt, $vtype, $media) = @_;
1338
1339     return if !$media;
1340
1341     my $etype;
1342     if ($media eq 'disk') {
1343         $etype = 'images';
1344     } elsif ($media eq 'cdrom') {
1345         $etype = 'iso';
1346     } else {
1347         die "internal error";
1348     }
1349
1350     return if ($vtype eq $etype);
1351
1352     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1353 }
1354
1355 sub cleanup_drive_path {
1356     my ($opt, $storecfg, $drive) = @_;
1357
1358     # try to convert filesystem paths to volume IDs
1359
1360     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1361         ($drive->{file} !~ m|^/dev/.+|) &&
1362         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1363         ($drive->{file} !~ m/^\d+$/)) {
1364         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1365         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1366             if !$vtype;
1367         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1368         verify_media_type($opt, $vtype, $drive->{media});
1369         $drive->{file} = $volid;
1370     }
1371
1372     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1373 }
1374
1375 sub parse_hotplug_features {
1376     my ($data) = @_;
1377
1378     my $res = {};
1379
1380     return $res if $data eq '0';
1381
1382     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1383
1384     foreach my $feature (PVE::Tools::split_list($data)) {
1385         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1386             $res->{$1} = 1;
1387         } else {
1388             die "invalid hotplug feature '$feature'\n";
1389         }
1390     }
1391     return $res;
1392 }
1393
1394 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1395 sub pve_verify_hotplug_features {
1396     my ($value, $noerr) = @_;
1397
1398     return $value if parse_hotplug_features($value);
1399
1400     return if $noerr;
1401
1402     die "unable to parse hotplug option\n";
1403 }
1404
1405 sub scsi_inquiry {
1406     my($fh, $noerr) = @_;
1407
1408     my $SG_IO = 0x2285;
1409     my $SG_GET_VERSION_NUM = 0x2282;
1410
1411     my $versionbuf = "\x00" x 8;
1412     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1413     if (!$ret) {
1414         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1415         return;
1416     }
1417     my $version = unpack("I", $versionbuf);
1418     if ($version < 30000) {
1419         die "scsi generic interface too old\n"  if !$noerr;
1420         return;
1421     }
1422
1423     my $buf = "\x00" x 36;
1424     my $sensebuf = "\x00" x 8;
1425     my $cmd = pack("C x3 C x1", 0x12, 36);
1426
1427     # see /usr/include/scsi/sg.h
1428     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1429
1430     my $packet = pack(
1431         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1432     );
1433
1434     $ret = ioctl($fh, $SG_IO, $packet);
1435     if (!$ret) {
1436         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1437         return;
1438     }
1439
1440     my @res = unpack($sg_io_hdr_t, $packet);
1441     if ($res[17] || $res[18]) {
1442         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1443         return;
1444     }
1445
1446     my $res = {};
1447     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1448
1449     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1450     $res->{type} &= 0x1F;
1451
1452     return $res;
1453 }
1454
1455 sub path_is_scsi {
1456     my ($path) = @_;
1457
1458     my $fh = IO::File->new("+<$path") || return;
1459     my $res = scsi_inquiry($fh, 1);
1460     close($fh);
1461
1462     return $res;
1463 }
1464
1465 sub print_tabletdevice_full {
1466     my ($conf, $arch) = @_;
1467
1468     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1469
1470     # we use uhci for old VMs because tablet driver was buggy in older qemu
1471     my $usbbus;
1472     if ($q35 || $arch eq 'aarch64') {
1473         $usbbus = 'ehci';
1474     } else {
1475         $usbbus = 'uhci';
1476     }
1477
1478     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1479 }
1480
1481 sub print_keyboarddevice_full {
1482     my ($conf, $arch) = @_;
1483
1484     return if $arch ne 'aarch64';
1485
1486     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1487 }
1488
1489 my sub get_drive_id {
1490     my ($drive) = @_;
1491     return "$drive->{interface}$drive->{index}";
1492 }
1493
1494 sub print_drivedevice_full {
1495     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1496
1497     my $device = '';
1498     my $maxdev = 0;
1499
1500     my $drive_id = get_drive_id($drive);
1501     if ($drive->{interface} eq 'virtio') {
1502         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1503         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1504         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1505     } elsif ($drive->{interface} eq 'scsi') {
1506
1507         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1508         my $unit = $drive->{index} % $maxdev;
1509         my $devicetype = 'hd';
1510         my $path = '';
1511         if (drive_is_cdrom($drive)) {
1512             $devicetype = 'cd';
1513         } else {
1514             if ($drive->{file} =~ m|^/|) {
1515                 $path = $drive->{file};
1516                 if (my $info = path_is_scsi($path)) {
1517                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1518                         $devicetype = 'block';
1519                     } elsif ($info->{type} == 1) { # tape
1520                         $devicetype = 'generic';
1521                     }
1522                 }
1523             } else {
1524                  $path = PVE::Storage::path($storecfg, $drive->{file});
1525             }
1526
1527             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1528             my $version = extract_version($machine_type, kvm_user_version());
1529             if ($path =~ m/^iscsi\:\/\// &&
1530                !min_version($version, 4, 1)) {
1531                 $devicetype = 'generic';
1532             }
1533         }
1534
1535         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1536             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1537         } else {
1538             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1539                 .",lun=$drive->{index}";
1540         }
1541         $device .= ",drive=drive-$drive_id,id=$drive_id";
1542
1543         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1544             $device .= ",rotation_rate=1";
1545         }
1546         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1547
1548     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1549         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1550         my $controller = int($drive->{index} / $maxdev);
1551         my $unit = $drive->{index} % $maxdev;
1552         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1553
1554         $device = "ide-$devicetype";
1555         if ($drive->{interface} eq 'ide') {
1556             $device .= ",bus=ide.$controller,unit=$unit";
1557         } else {
1558             $device .= ",bus=ahci$controller.$unit";
1559         }
1560         $device .= ",drive=drive-$drive_id,id=$drive_id";
1561
1562         if ($devicetype eq 'hd') {
1563             if (my $model = $drive->{model}) {
1564                 $model = URI::Escape::uri_unescape($model);
1565                 $device .= ",model=$model";
1566             }
1567             if ($drive->{ssd}) {
1568                 $device .= ",rotation_rate=1";
1569             }
1570         }
1571         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1572     } elsif ($drive->{interface} eq 'usb') {
1573         die "implement me";
1574         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1575     } else {
1576         die "unsupported interface type";
1577     }
1578
1579     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1580
1581     if (my $serial = $drive->{serial}) {
1582         $serial = URI::Escape::uri_unescape($serial);
1583         $device .= ",serial=$serial";
1584     }
1585
1586
1587     return $device;
1588 }
1589
1590 sub get_initiator_name {
1591     my $initiator;
1592
1593     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1594     while (defined(my $line = <$fh>)) {
1595         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1596         $initiator = $1;
1597         last;
1598     }
1599     $fh->close();
1600
1601     return $initiator;
1602 }
1603
1604 sub print_drive_commandline_full {
1605     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1606
1607     my $path;
1608     my $volid = $drive->{file};
1609     my $format = $drive->{format};
1610     my $drive_id = get_drive_id($drive);
1611
1612     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1613     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1614
1615     if (drive_is_cdrom($drive)) {
1616         $path = get_iso_path($storecfg, $vmid, $volid);
1617         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1618     } else {
1619         if ($storeid) {
1620             $path = PVE::Storage::path($storecfg, $volid);
1621             $format //= qemu_img_format($scfg, $volname);
1622         } else {
1623             $path = $volid;
1624             $format //= "raw";
1625         }
1626    }
1627
1628    my $is_rbd = $path =~ m/^rbd:/;
1629
1630     my $opts = '';
1631     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1632     foreach my $o (@qemu_drive_options) {
1633         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1634     }
1635
1636     # snapshot only accepts on|off
1637     if (defined($drive->{snapshot})) {
1638         my $v = $drive->{snapshot} ? 'on' : 'off';
1639         $opts .= ",snapshot=$v";
1640     }
1641
1642     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1643         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1644     }
1645
1646     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1647         my ($dir, $qmpname) = @$type;
1648         if (my $v = $drive->{"mbps$dir"}) {
1649             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1650         }
1651         if (my $v = $drive->{"mbps${dir}_max"}) {
1652             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1653         }
1654         if (my $v = $drive->{"bps${dir}_max_length"}) {
1655             $opts .= ",throttling.bps$qmpname-max-length=$v";
1656         }
1657         if (my $v = $drive->{"iops${dir}"}) {
1658             $opts .= ",throttling.iops$qmpname=$v";
1659         }
1660         if (my $v = $drive->{"iops${dir}_max"}) {
1661             $opts .= ",throttling.iops$qmpname-max=$v";
1662         }
1663         if (my $v = $drive->{"iops${dir}_max_length"}) {
1664             $opts .= ",throttling.iops$qmpname-max-length=$v";
1665         }
1666     }
1667
1668     if ($pbs_name) {
1669         $format = "rbd" if $is_rbd;
1670         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1671             if !$format;
1672         $opts .= ",format=alloc-track,file.driver=$format";
1673     } elsif ($format) {
1674         $opts .= ",format=$format";
1675     }
1676
1677     my $cache_direct = 0;
1678
1679     if (my $cache = $drive->{cache}) {
1680         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1681     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1682         $opts .= ",cache=none";
1683         $cache_direct = 1;
1684     }
1685
1686     # io_uring with cache mode writeback or writethrough on krbd will hang...
1687     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1688
1689     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1690     # sometimes, just plain disable...
1691     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1692
1693     # io_uring causes problems when used with CIFS since kernel 5.15
1694     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1695     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1696
1697     if (!$drive->{aio}) {
1698         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1699             # io_uring supports all cache modes
1700             $opts .= ",aio=io_uring";
1701         } else {
1702             # aio native works only with O_DIRECT
1703             if($cache_direct) {
1704                 $opts .= ",aio=native";
1705             } else {
1706                 $opts .= ",aio=threads";
1707             }
1708         }
1709     }
1710
1711     if (!drive_is_cdrom($drive)) {
1712         my $detectzeroes;
1713         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1714             $detectzeroes = 'off';
1715         } elsif ($drive->{discard}) {
1716             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1717         } else {
1718             # This used to be our default with discard not being specified:
1719             $detectzeroes = 'on';
1720         }
1721
1722         # note: 'detect-zeroes' works per blockdev and we want it to persist
1723         # after the alloc-track is removed, so put it on 'file' directly
1724         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1725         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1726     }
1727
1728     if ($pbs_name) {
1729         $opts .= ",backing=$pbs_name";
1730         $opts .= ",auto-remove=on";
1731     }
1732
1733     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1734     my $file_param = "file";
1735     if ($pbs_name) {
1736         # non-rbd drivers require the underlying file to be a seperate block
1737         # node, so add a second .file indirection
1738         $file_param .= ".file" if !$is_rbd;
1739         $file_param .= ".filename";
1740     }
1741     my $pathinfo = $path ? "$file_param=$path," : '';
1742
1743     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1744 }
1745
1746 sub print_pbs_blockdev {
1747     my ($pbs_conf, $pbs_name) = @_;
1748     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1749     $blockdev .= ",repository=$pbs_conf->{repository}";
1750     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1751     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1752     $blockdev .= ",archive=$pbs_conf->{archive}";
1753     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1754     return $blockdev;
1755 }
1756
1757 sub print_netdevice_full {
1758     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1759
1760     my $device = $net->{model};
1761     if ($net->{model} eq 'virtio') {
1762          $device = 'virtio-net-pci';
1763      };
1764
1765     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1766     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1767     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1768         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1769         # and out of each queue plus one config interrupt and control vector queue
1770         my $vectors = $net->{queues} * 2 + 2;
1771         $tmpstr .= ",vectors=$vectors,mq=on";
1772         if (min_version($machine_version, 7, 1)) {
1773             $tmpstr .= ",packed=on";
1774         }
1775     }
1776
1777     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1778         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1779     }
1780
1781     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1782
1783     if (my $mtu = $net->{mtu}) {
1784         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1785             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1786             if ($mtu == 1) {
1787                  $mtu = $bridge_mtu;
1788             } elsif ($mtu < 576) {
1789                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1790             } elsif ($mtu > $bridge_mtu) {
1791                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1792             }
1793             $tmpstr .= ",host_mtu=$mtu";
1794         } else {
1795             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1796         }
1797     }
1798
1799     if ($use_old_bios_files) {
1800         my $romfile;
1801         if ($device eq 'virtio-net-pci') {
1802             $romfile = 'pxe-virtio.rom';
1803         } elsif ($device eq 'e1000') {
1804             $romfile = 'pxe-e1000.rom';
1805         } elsif ($device eq 'e1000e') {
1806             $romfile = 'pxe-e1000e.rom';
1807         } elsif ($device eq 'ne2k') {
1808             $romfile = 'pxe-ne2k_pci.rom';
1809         } elsif ($device eq 'pcnet') {
1810             $romfile = 'pxe-pcnet.rom';
1811         } elsif ($device eq 'rtl8139') {
1812             $romfile = 'pxe-rtl8139.rom';
1813         }
1814         $tmpstr .= ",romfile=$romfile" if $romfile;
1815     }
1816
1817     return $tmpstr;
1818 }
1819
1820 sub print_netdev_full {
1821     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1822
1823     my $i = '';
1824     if ($netid =~ m/^net(\d+)$/) {
1825         $i = int($1);
1826     }
1827
1828     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1829
1830     my $ifname = "tap${vmid}i$i";
1831
1832     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1833     die "interface name '$ifname' is too long (max 15 character)\n"
1834         if length($ifname) >= 16;
1835
1836     my $vhostparam = '';
1837     if (is_native($arch)) {
1838         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1839     }
1840
1841     my $vmname = $conf->{name} || "vm$vmid";
1842
1843     my $netdev = "";
1844     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1845
1846     if ($net->{bridge}) {
1847         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1848             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1849     } else {
1850         $netdev = "type=user,id=$netid,hostname=$vmname";
1851     }
1852
1853     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1854
1855     return $netdev;
1856 }
1857
1858 my $vga_map = {
1859     'cirrus' => 'cirrus-vga',
1860     'std' => 'VGA',
1861     'vmware' => 'vmware-svga',
1862     'virtio' => 'virtio-vga',
1863     'virtio-gl' => 'virtio-vga-gl',
1864 };
1865
1866 sub print_vga_device {
1867     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1868
1869     my $type = $vga_map->{$vga->{type}};
1870     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1871         $type = 'virtio-gpu';
1872     }
1873     my $vgamem_mb = $vga->{memory};
1874
1875     my $max_outputs = '';
1876     if ($qxlnum) {
1877         $type = $id ? 'qxl' : 'qxl-vga';
1878
1879         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1880             # set max outputs so linux can have up to 4 qxl displays with one device
1881             if (min_version($machine_version, 4, 1)) {
1882                 $max_outputs = ",max_outputs=4";
1883             }
1884         }
1885     }
1886
1887     die "no devicetype for $vga->{type}\n" if !$type;
1888
1889     my $memory = "";
1890     if ($vgamem_mb) {
1891         if ($vga->{type} =~ /^virtio/) {
1892             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1893             $memory = ",max_hostmem=$bytes";
1894         } elsif ($qxlnum) {
1895             # from https://www.spice-space.org/multiple-monitors.html
1896             $memory = ",vgamem_mb=$vga->{memory}";
1897             my $ram = $vgamem_mb * 4;
1898             my $vram = $vgamem_mb * 2;
1899             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1900         } else {
1901             $memory = ",vgamem_mb=$vga->{memory}";
1902         }
1903     } elsif ($qxlnum && $id) {
1904         $memory = ",ram_size=67108864,vram_size=33554432";
1905     }
1906
1907     my $edidoff = "";
1908     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1909         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1910     }
1911
1912     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1913     my $vgaid = "vga" . ($id // '');
1914     my $pciaddr;
1915     if ($q35 && $vgaid eq 'vga') {
1916         # the first display uses pcie.0 bus on q35 machines
1917         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1918     } else {
1919         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1920     }
1921
1922     if ($vga->{type} eq 'virtio-gl') {
1923         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1924         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1925             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1926
1927         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1928             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1929     }
1930
1931     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1932 }
1933
1934 sub parse_number_sets {
1935     my ($set) = @_;
1936     my $res = [];
1937     foreach my $part (split(/;/, $set)) {
1938         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1939             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1940             push @$res, [ $1, $2 ];
1941         } else {
1942             die "invalid range: $part\n";
1943         }
1944     }
1945     return $res;
1946 }
1947
1948 sub parse_numa {
1949     my ($data) = @_;
1950
1951     my $res = parse_property_string($numa_fmt, $data);
1952     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1953     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1954     return $res;
1955 }
1956
1957 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1958 sub parse_net {
1959     my ($data, $disable_mac_autogen) = @_;
1960
1961     my $res = eval { parse_property_string($net_fmt, $data) };
1962     if ($@) {
1963         warn $@;
1964         return;
1965     }
1966     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1967         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1968         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1969     }
1970     return $res;
1971 }
1972
1973 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1974 sub parse_ipconfig {
1975     my ($data) = @_;
1976
1977     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1978     if ($@) {
1979         warn $@;
1980         return;
1981     }
1982
1983     if ($res->{gw} && !$res->{ip}) {
1984         warn 'gateway specified without specifying an IP address';
1985         return;
1986     }
1987     if ($res->{gw6} && !$res->{ip6}) {
1988         warn 'IPv6 gateway specified without specifying an IPv6 address';
1989         return;
1990     }
1991     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1992         warn 'gateway specified together with DHCP';
1993         return;
1994     }
1995     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1996         # gw6 + auto/dhcp
1997         warn "IPv6 gateway specified together with $res->{ip6} address";
1998         return;
1999     }
2000
2001     if (!$res->{ip} && !$res->{ip6}) {
2002         return { ip => 'dhcp', ip6 => 'dhcp' };
2003     }
2004
2005     return $res;
2006 }
2007
2008 sub print_net {
2009     my $net = shift;
2010
2011     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2012 }
2013
2014 sub add_random_macs {
2015     my ($settings) = @_;
2016
2017     foreach my $opt (keys %$settings) {
2018         next if $opt !~ m/^net(\d+)$/;
2019         my $net = parse_net($settings->{$opt});
2020         next if !$net;
2021         $settings->{$opt} = print_net($net);
2022     }
2023 }
2024
2025 sub vm_is_volid_owner {
2026     my ($storecfg, $vmid, $volid) = @_;
2027
2028     if ($volid !~  m|^/|) {
2029         my ($path, $owner);
2030         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2031         if ($owner && ($owner == $vmid)) {
2032             return 1;
2033         }
2034     }
2035
2036     return;
2037 }
2038
2039 sub vmconfig_register_unused_drive {
2040     my ($storecfg, $vmid, $conf, $drive) = @_;
2041
2042     if (drive_is_cloudinit($drive)) {
2043         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2044         warn $@ if $@;
2045         delete $conf->{cloudinit};
2046     } elsif (!drive_is_cdrom($drive)) {
2047         my $volid = $drive->{file};
2048         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2049             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2050         }
2051     }
2052 }
2053
2054 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2055 my $smbios1_fmt = {
2056     uuid => {
2057         type => 'string',
2058         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2059         format_description => 'UUID',
2060         description => "Set SMBIOS1 UUID.",
2061         optional => 1,
2062     },
2063     version => {
2064         type => 'string',
2065         pattern => '[A-Za-z0-9+\/]+={0,2}',
2066         format_description => 'Base64 encoded string',
2067         description => "Set SMBIOS1 version.",
2068         optional => 1,
2069     },
2070     serial => {
2071         type => 'string',
2072         pattern => '[A-Za-z0-9+\/]+={0,2}',
2073         format_description => 'Base64 encoded string',
2074         description => "Set SMBIOS1 serial number.",
2075         optional => 1,
2076     },
2077     manufacturer => {
2078         type => 'string',
2079         pattern => '[A-Za-z0-9+\/]+={0,2}',
2080         format_description => 'Base64 encoded string',
2081         description => "Set SMBIOS1 manufacturer.",
2082         optional => 1,
2083     },
2084     product => {
2085         type => 'string',
2086         pattern => '[A-Za-z0-9+\/]+={0,2}',
2087         format_description => 'Base64 encoded string',
2088         description => "Set SMBIOS1 product ID.",
2089         optional => 1,
2090     },
2091     sku => {
2092         type => 'string',
2093         pattern => '[A-Za-z0-9+\/]+={0,2}',
2094         format_description => 'Base64 encoded string',
2095         description => "Set SMBIOS1 SKU string.",
2096         optional => 1,
2097     },
2098     family => {
2099         type => 'string',
2100         pattern => '[A-Za-z0-9+\/]+={0,2}',
2101         format_description => 'Base64 encoded string',
2102         description => "Set SMBIOS1 family string.",
2103         optional => 1,
2104     },
2105     base64 => {
2106         type => 'boolean',
2107         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2108         optional => 1,
2109     },
2110 };
2111
2112 sub parse_smbios1 {
2113     my ($data) = @_;
2114
2115     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2116     warn $@ if $@;
2117     return $res;
2118 }
2119
2120 sub print_smbios1 {
2121     my ($smbios1) = @_;
2122     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2123 }
2124
2125 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2126
2127 sub parse_watchdog {
2128     my ($value) = @_;
2129
2130     return if !$value;
2131
2132     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2133     warn $@ if $@;
2134     return $res;
2135 }
2136
2137 sub parse_guest_agent {
2138     my ($conf) = @_;
2139
2140     return {} if !defined($conf->{agent});
2141
2142     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2143     warn $@ if $@;
2144
2145     # if the agent is disabled ignore the other potentially set properties
2146     return {} if !$res->{enabled};
2147     return $res;
2148 }
2149
2150 sub get_qga_key {
2151     my ($conf, $key) = @_;
2152     return undef if !defined($conf->{agent});
2153
2154     my $agent = parse_guest_agent($conf);
2155     return $agent->{$key};
2156 }
2157
2158 sub parse_vga {
2159     my ($value) = @_;
2160
2161     return {} if !$value;
2162     my $res = eval { parse_property_string($vga_fmt, $value) };
2163     warn $@ if $@;
2164     return $res;
2165 }
2166
2167 sub parse_rng {
2168     my ($value) = @_;
2169
2170     return if !$value;
2171
2172     my $res = eval { parse_property_string($rng_fmt, $value) };
2173     warn $@ if $@;
2174     return $res;
2175 }
2176
2177 sub parse_meta_info {
2178     my ($value) = @_;
2179
2180     return if !$value;
2181
2182     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2183     warn $@ if $@;
2184     return $res;
2185 }
2186
2187 sub new_meta_info_string {
2188     my () = @_; # for now do not allow to override any value
2189
2190     return PVE::JSONSchema::print_property_string(
2191         {
2192             'creation-qemu' => kvm_user_version(),
2193             ctime => "". int(time()),
2194         },
2195         $meta_info_fmt
2196     );
2197 }
2198
2199 sub qemu_created_version_fixups {
2200     my ($conf, $forcemachine, $kvmver) = @_;
2201
2202     my $meta = parse_meta_info($conf->{meta}) // {};
2203     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2204
2205     # check if we need to apply some handling for VMs that always use the latest machine version but
2206     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2207     # would be required (we do not want to pin machine version for non-windows OS type)
2208     if (
2209         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2210         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2211         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2212         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2213     ) {
2214         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2215         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2216             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2217             # and thus with the predictable interface naming of systemd
2218             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2219         }
2220     }
2221     return;
2222 }
2223
2224 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2225 sub verify_usb_device {
2226     my ($value, $noerr) = @_;
2227
2228     return $value if parse_usb_device($value);
2229
2230     return if $noerr;
2231
2232     die "unable to parse usb device\n";
2233 }
2234
2235 # add JSON properties for create and set function
2236 sub json_config_properties {
2237     my ($prop, $with_disk_alloc) = @_;
2238
2239     my $skip_json_config_opts = {
2240         parent => 1,
2241         snaptime => 1,
2242         vmstate => 1,
2243         runningmachine => 1,
2244         runningcpu => 1,
2245         meta => 1,
2246     };
2247
2248     foreach my $opt (keys %$confdesc) {
2249         next if $skip_json_config_opts->{$opt};
2250
2251         if ($with_disk_alloc && is_valid_drivename($opt)) {
2252             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2253         } else {
2254             $prop->{$opt} = $confdesc->{$opt};
2255         }
2256     }
2257
2258     return $prop;
2259 }
2260
2261 # Properties that we can read from an OVF file
2262 sub json_ovf_properties {
2263     my $prop = {};
2264
2265     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2266         $prop->{$device} = {
2267             type => 'string',
2268             format => 'pve-volume-id-or-absolute-path',
2269             description => "Disk image that gets imported to $device",
2270             optional => 1,
2271         };
2272     }
2273
2274     $prop->{cores} = {
2275         type => 'integer',
2276         description => "The number of CPU cores.",
2277         optional => 1,
2278     };
2279     $prop->{memory} = {
2280         type => 'integer',
2281         description => "Amount of RAM for the VM in MB.",
2282         optional => 1,
2283     };
2284     $prop->{name} = {
2285         type => 'string',
2286         description => "Name of the VM.",
2287         optional => 1,
2288     };
2289
2290     return $prop;
2291 }
2292
2293 # return copy of $confdesc_cloudinit to generate documentation
2294 sub cloudinit_config_properties {
2295
2296     return dclone($confdesc_cloudinit);
2297 }
2298
2299 sub cloudinit_pending_properties {
2300     my $p = {
2301         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2302         name => 1,
2303     };
2304     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2305     return $p;
2306 }
2307
2308 sub check_type {
2309     my ($key, $value) = @_;
2310
2311     die "unknown setting '$key'\n" if !$confdesc->{$key};
2312
2313     my $type = $confdesc->{$key}->{type};
2314
2315     if (!defined($value)) {
2316         die "got undefined value\n";
2317     }
2318
2319     if ($value =~ m/[\n\r]/) {
2320         die "property contains a line feed\n";
2321     }
2322
2323     if ($type eq 'boolean') {
2324         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2325         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2326         die "type check ('boolean') failed - got '$value'\n";
2327     } elsif ($type eq 'integer') {
2328         return int($1) if $value =~ m/^(\d+)$/;
2329         die "type check ('integer') failed - got '$value'\n";
2330     } elsif ($type eq 'number') {
2331         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2332         die "type check ('number') failed - got '$value'\n";
2333     } elsif ($type eq 'string') {
2334         if (my $fmt = $confdesc->{$key}->{format}) {
2335             PVE::JSONSchema::check_format($fmt, $value);
2336             return $value;
2337         }
2338         $value =~ s/^\"(.*)\"$/$1/;
2339         return $value;
2340     } else {
2341         die "internal error"
2342     }
2343 }
2344
2345 sub destroy_vm {
2346     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2347
2348     my $conf = PVE::QemuConfig->load_config($vmid);
2349
2350     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2351         PVE::QemuConfig->check_lock($conf);
2352     }
2353
2354     if ($conf->{template}) {
2355         # check if any base image is still used by a linked clone
2356         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2357                 my ($ds, $drive) = @_;
2358                 return if drive_is_cdrom($drive);
2359
2360                 my $volid = $drive->{file};
2361                 return if !$volid || $volid =~ m|^/|;
2362
2363                 die "base volume '$volid' is still in use by linked cloned\n"
2364                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2365
2366         });
2367     }
2368
2369     my $volids = {};
2370     my $remove_owned_drive = sub {
2371         my ($ds, $drive) = @_;
2372         return if drive_is_cdrom($drive, 1);
2373
2374         my $volid = $drive->{file};
2375         return if !$volid || $volid =~ m|^/|;
2376         return if $volids->{$volid};
2377
2378         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2379         return if !$path || !$owner || ($owner != $vmid);
2380
2381         $volids->{$volid} = 1;
2382         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2383         warn "Could not remove disk '$volid', check manually: $@" if $@;
2384     };
2385
2386     # only remove disks owned by this VM (referenced in the config)
2387     my $include_opts = {
2388         include_unused => 1,
2389         extra_keys => ['vmstate'],
2390     };
2391     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2392
2393     for my $snap (values %{$conf->{snapshots}}) {
2394         next if !defined($snap->{vmstate});
2395         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2396         next if !defined($drive);
2397         $remove_owned_drive->('vmstate', $drive);
2398     }
2399
2400     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2401
2402     if ($purge_unreferenced) { # also remove unreferenced disk
2403         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2404         PVE::Storage::foreach_volid($vmdisks, sub {
2405             my ($volid, $sid, $volname, $d) = @_;
2406             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2407             warn $@ if $@;
2408         });
2409     }
2410
2411     if (defined $replacement_conf) {
2412         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2413     } else {
2414         PVE::QemuConfig->destroy_config($vmid);
2415     }
2416 }
2417
2418 sub parse_vm_config {
2419     my ($filename, $raw, $strict) = @_;
2420
2421     return if !defined($raw);
2422
2423     my $res = {
2424         digest => Digest::SHA::sha1_hex($raw),
2425         snapshots => {},
2426         pending => {},
2427         cloudinit => {},
2428     };
2429
2430     my $handle_error = sub {
2431         my ($msg) = @_;
2432
2433         if ($strict) {
2434             die $msg;
2435         } else {
2436             warn $msg;
2437         }
2438     };
2439
2440     $filename =~ m|/qemu-server/(\d+)\.conf$|
2441         || die "got strange filename '$filename'";
2442
2443     my $vmid = $1;
2444
2445     my $conf = $res;
2446     my $descr;
2447     my $finish_description = sub {
2448         if (defined($descr)) {
2449             $descr =~ s/\s+$//;
2450             $conf->{description} = $descr;
2451         }
2452         $descr = undef;
2453     };
2454     my $section = '';
2455
2456     my @lines = split(/\n/, $raw);
2457     foreach my $line (@lines) {
2458         next if $line =~ m/^\s*$/;
2459
2460         if ($line =~ m/^\[PENDING\]\s*$/i) {
2461             $section = 'pending';
2462             $finish_description->();
2463             $conf = $res->{$section} = {};
2464             next;
2465         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2466             $section = 'cloudinit';
2467             $finish_description->();
2468             $conf = $res->{$section} = {};
2469             next;
2470
2471         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2472             $section = $1;
2473             $finish_description->();
2474             $conf = $res->{snapshots}->{$section} = {};
2475             next;
2476         }
2477
2478         if ($line =~ m/^\#(.*)$/) {
2479             $descr = '' if !defined($descr);
2480             $descr .= PVE::Tools::decode_text($1) . "\n";
2481             next;
2482         }
2483
2484         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2485             $descr = '' if !defined($descr);
2486             $descr .= PVE::Tools::decode_text($2);
2487         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2488             $conf->{snapstate} = $1;
2489         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2490             my $key = $1;
2491             my $value = $2;
2492             $conf->{$key} = $value;
2493         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2494             my $value = $1;
2495             if ($section eq 'pending') {
2496                 $conf->{delete} = $value; # we parse this later
2497             } else {
2498                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2499             }
2500         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2501             my $key = $1;
2502             my $value = $2;
2503             if ($section eq 'cloudinit') {
2504                 # ignore validation only used for informative purpose
2505                 $conf->{$key} = $value;
2506                 next;
2507             }
2508             eval { $value = check_type($key, $value); };
2509             if ($@) {
2510                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2511             } else {
2512                 $key = 'ide2' if $key eq 'cdrom';
2513                 my $fmt = $confdesc->{$key}->{format};
2514                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2515                     my $v = parse_drive($key, $value);
2516                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2517                         $v->{file} = $volid;
2518                         $value = print_drive($v);
2519                     } else {
2520                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2521                         next;
2522                     }
2523                 }
2524
2525                 $conf->{$key} = $value;
2526             }
2527         } else {
2528             $handle_error->("vm $vmid - unable to parse config: $line\n");
2529         }
2530     }
2531
2532     $finish_description->();
2533     delete $res->{snapstate}; # just to be sure
2534
2535     return $res;
2536 }
2537
2538 sub write_vm_config {
2539     my ($filename, $conf) = @_;
2540
2541     delete $conf->{snapstate}; # just to be sure
2542
2543     if ($conf->{cdrom}) {
2544         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2545         $conf->{ide2} = $conf->{cdrom};
2546         delete $conf->{cdrom};
2547     }
2548
2549     # we do not use 'smp' any longer
2550     if ($conf->{sockets}) {
2551         delete $conf->{smp};
2552     } elsif ($conf->{smp}) {
2553         $conf->{sockets} = $conf->{smp};
2554         delete $conf->{cores};
2555         delete $conf->{smp};
2556     }
2557
2558     my $used_volids = {};
2559
2560     my $cleanup_config = sub {
2561         my ($cref, $pending, $snapname) = @_;
2562
2563         foreach my $key (keys %$cref) {
2564             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2565                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2566             my $value = $cref->{$key};
2567             if ($key eq 'delete') {
2568                 die "propertry 'delete' is only allowed in [PENDING]\n"
2569                     if !$pending;
2570                 # fixme: check syntax?
2571                 next;
2572             }
2573             eval { $value = check_type($key, $value); };
2574             die "unable to parse value of '$key' - $@" if $@;
2575
2576             $cref->{$key} = $value;
2577
2578             if (!$snapname && is_valid_drivename($key)) {
2579                 my $drive = parse_drive($key, $value);
2580                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2581             }
2582         }
2583     };
2584
2585     &$cleanup_config($conf);
2586
2587     &$cleanup_config($conf->{pending}, 1);
2588
2589     foreach my $snapname (keys %{$conf->{snapshots}}) {
2590         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2591         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2592     }
2593
2594     # remove 'unusedX' settings if we re-add a volume
2595     foreach my $key (keys %$conf) {
2596         my $value = $conf->{$key};
2597         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2598             delete $conf->{$key};
2599         }
2600     }
2601
2602     my $generate_raw_config = sub {
2603         my ($conf, $pending) = @_;
2604
2605         my $raw = '';
2606
2607         # add description as comment to top of file
2608         if (defined(my $descr = $conf->{description})) {
2609             if ($descr) {
2610                 foreach my $cl (split(/\n/, $descr)) {
2611                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2612                 }
2613             } else {
2614                 $raw .= "#\n" if $pending;
2615             }
2616         }
2617
2618         foreach my $key (sort keys %$conf) {
2619             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2620             $raw .= "$key: $conf->{$key}\n";
2621         }
2622         return $raw;
2623     };
2624
2625     my $raw = &$generate_raw_config($conf);
2626
2627     if (scalar(keys %{$conf->{pending}})){
2628         $raw .= "\n[PENDING]\n";
2629         $raw .= &$generate_raw_config($conf->{pending}, 1);
2630     }
2631
2632     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2633         $raw .= "\n[special:cloudinit]\n";
2634         $raw .= &$generate_raw_config($conf->{cloudinit});
2635     }
2636
2637     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2638         $raw .= "\n[$snapname]\n";
2639         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2640     }
2641
2642     return $raw;
2643 }
2644
2645 sub load_defaults {
2646
2647     my $res = {};
2648
2649     # we use static defaults from our JSON schema configuration
2650     foreach my $key (keys %$confdesc) {
2651         if (defined(my $default = $confdesc->{$key}->{default})) {
2652             $res->{$key} = $default;
2653         }
2654     }
2655
2656     return $res;
2657 }
2658
2659 sub config_list {
2660     my $vmlist = PVE::Cluster::get_vmlist();
2661     my $res = {};
2662     return $res if !$vmlist || !$vmlist->{ids};
2663     my $ids = $vmlist->{ids};
2664     my $nodename = nodename();
2665
2666     foreach my $vmid (keys %$ids) {
2667         my $d = $ids->{$vmid};
2668         next if !$d->{node} || $d->{node} ne $nodename;
2669         next if !$d->{type} || $d->{type} ne 'qemu';
2670         $res->{$vmid}->{exists} = 1;
2671     }
2672     return $res;
2673 }
2674
2675 # test if VM uses local resources (to prevent migration)
2676 sub check_local_resources {
2677     my ($conf, $noerr) = @_;
2678
2679     my @loc_res = ();
2680
2681     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2682     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2683
2684     push @loc_res, "ivshmem" if $conf->{ivshmem};
2685
2686     foreach my $k (keys %$conf) {
2687         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2688         # sockets are safe: they will recreated be on the target side post-migrate
2689         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2690         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2691     }
2692
2693     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2694
2695     return \@loc_res;
2696 }
2697
2698 # check if used storages are available on all nodes (use by migrate)
2699 sub check_storage_availability {
2700     my ($storecfg, $conf, $node) = @_;
2701
2702     PVE::QemuConfig->foreach_volume($conf, sub {
2703         my ($ds, $drive) = @_;
2704
2705         my $volid = $drive->{file};
2706         return if !$volid;
2707
2708         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2709         return if !$sid;
2710
2711         # check if storage is available on both nodes
2712         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2713         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2714
2715         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2716
2717         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2718             if !$scfg->{content}->{$vtype};
2719    });
2720 }
2721
2722 # list nodes where all VM images are available (used by has_feature API)
2723 sub shared_nodes {
2724     my ($conf, $storecfg) = @_;
2725
2726     my $nodelist = PVE::Cluster::get_nodelist();
2727     my $nodehash = { map { $_ => 1 } @$nodelist };
2728     my $nodename = nodename();
2729
2730     PVE::QemuConfig->foreach_volume($conf, sub {
2731         my ($ds, $drive) = @_;
2732
2733         my $volid = $drive->{file};
2734         return if !$volid;
2735
2736         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2737         if ($storeid) {
2738             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2739             if ($scfg->{disable}) {
2740                 $nodehash = {};
2741             } elsif (my $avail = $scfg->{nodes}) {
2742                 foreach my $node (keys %$nodehash) {
2743                     delete $nodehash->{$node} if !$avail->{$node};
2744                 }
2745             } elsif (!$scfg->{shared}) {
2746                 foreach my $node (keys %$nodehash) {
2747                     delete $nodehash->{$node} if $node ne $nodename
2748                 }
2749             }
2750         }
2751     });
2752
2753     return $nodehash
2754 }
2755
2756 sub check_local_storage_availability {
2757     my ($conf, $storecfg) = @_;
2758
2759     my $nodelist = PVE::Cluster::get_nodelist();
2760     my $nodehash = { map { $_ => {} } @$nodelist };
2761
2762     PVE::QemuConfig->foreach_volume($conf, sub {
2763         my ($ds, $drive) = @_;
2764
2765         my $volid = $drive->{file};
2766         return if !$volid;
2767
2768         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2769         if ($storeid) {
2770             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2771
2772             if ($scfg->{disable}) {
2773                 foreach my $node (keys %$nodehash) {
2774                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2775                 }
2776             } elsif (my $avail = $scfg->{nodes}) {
2777                 foreach my $node (keys %$nodehash) {
2778                     if (!$avail->{$node}) {
2779                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2780                     }
2781                 }
2782             }
2783         }
2784     });
2785
2786     foreach my $node (values %$nodehash) {
2787         if (my $unavail = $node->{unavailable_storages}) {
2788             $node->{unavailable_storages} = [ sort keys %$unavail ];
2789         }
2790     }
2791
2792     return $nodehash
2793 }
2794
2795 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2796 sub check_running {
2797     my ($vmid, $nocheck, $node) = @_;
2798
2799     # $nocheck is set when called during a migration, in which case the config
2800     # file might still or already reside on the *other* node
2801     # - because rename has already happened, and current node is source
2802     # - because rename hasn't happened yet, and current node is target
2803     # - because rename has happened, current node is target, but hasn't yet
2804     # processed it yet
2805     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2806     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2807 }
2808
2809 sub vzlist {
2810
2811     my $vzlist = config_list();
2812
2813     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2814
2815     while (defined(my $de = $fd->read)) {
2816         next if $de !~ m/^(\d+)\.pid$/;
2817         my $vmid = $1;
2818         next if !defined($vzlist->{$vmid});
2819         if (my $pid = check_running($vmid)) {
2820             $vzlist->{$vmid}->{pid} = $pid;
2821         }
2822     }
2823
2824     return $vzlist;
2825 }
2826
2827 our $vmstatus_return_properties = {
2828     vmid => get_standard_option('pve-vmid'),
2829     status => {
2830         description => "QEMU process status.",
2831         type => 'string',
2832         enum => ['stopped', 'running'],
2833     },
2834     maxmem => {
2835         description => "Maximum memory in bytes.",
2836         type => 'integer',
2837         optional => 1,
2838         renderer => 'bytes',
2839     },
2840     maxdisk => {
2841         description => "Root disk size in bytes.",
2842         type => 'integer',
2843         optional => 1,
2844         renderer => 'bytes',
2845     },
2846     name => {
2847         description => "VM name.",
2848         type => 'string',
2849         optional => 1,
2850     },
2851     qmpstatus => {
2852         description => "QEMU QMP agent status.",
2853         type => 'string',
2854         optional => 1,
2855     },
2856     pid => {
2857         description => "PID of running qemu process.",
2858         type => 'integer',
2859         optional => 1,
2860     },
2861     uptime => {
2862         description => "Uptime.",
2863         type => 'integer',
2864         optional => 1,
2865         renderer => 'duration',
2866     },
2867     cpus => {
2868         description => "Maximum usable CPUs.",
2869         type => 'number',
2870         optional => 1,
2871     },
2872     lock => {
2873         description => "The current config lock, if any.",
2874         type => 'string',
2875         optional => 1,
2876     },
2877     tags => {
2878         description  => "The current configured tags, if any",
2879         type => 'string',
2880         optional => 1,
2881     },
2882     'running-machine' => {
2883         description => "The currently running machine type (if running).",
2884         type => 'string',
2885         optional => 1,
2886     },
2887     'running-qemu' => {
2888         description => "The currently running QEMU version (if running).",
2889         type => 'string',
2890         optional => 1,
2891     },
2892 };
2893
2894 my $last_proc_pid_stat;
2895
2896 # get VM status information
2897 # This must be fast and should not block ($full == false)
2898 # We only query KVM using QMP if $full == true (this can be slow)
2899 sub vmstatus {
2900     my ($opt_vmid, $full) = @_;
2901
2902     my $res = {};
2903
2904     my $storecfg = PVE::Storage::config();
2905
2906     my $list = vzlist();
2907     my $defaults = load_defaults();
2908
2909     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2910
2911     my $cpucount = $cpuinfo->{cpus} || 1;
2912
2913     foreach my $vmid (keys %$list) {
2914         next if $opt_vmid && ($vmid ne $opt_vmid);
2915
2916         my $conf = PVE::QemuConfig->load_config($vmid);
2917
2918         my $d = { vmid => int($vmid) };
2919         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2920
2921         # fixme: better status?
2922         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2923
2924         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2925         if (defined($size)) {
2926             $d->{disk} = 0; # no info available
2927             $d->{maxdisk} = $size;
2928         } else {
2929             $d->{disk} = 0;
2930             $d->{maxdisk} = 0;
2931         }
2932
2933         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2934             * ($conf->{cores} || $defaults->{cores});
2935         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2936         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2937
2938         $d->{name} = $conf->{name} || "VM $vmid";
2939         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2940             : $defaults->{memory}*(1024*1024);
2941
2942         if ($conf->{balloon}) {
2943             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2944             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2945                 : $defaults->{shares};
2946         }
2947
2948         $d->{uptime} = 0;
2949         $d->{cpu} = 0;
2950         $d->{mem} = 0;
2951
2952         $d->{netout} = 0;
2953         $d->{netin} = 0;
2954
2955         $d->{diskread} = 0;
2956         $d->{diskwrite} = 0;
2957
2958         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2959
2960         $d->{serial} = 1 if conf_has_serial($conf);
2961         $d->{lock} = $conf->{lock} if $conf->{lock};
2962         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2963
2964         $res->{$vmid} = $d;
2965     }
2966
2967     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2968     foreach my $dev (keys %$netdev) {
2969         next if $dev !~ m/^tap([1-9]\d*)i/;
2970         my $vmid = $1;
2971         my $d = $res->{$vmid};
2972         next if !$d;
2973
2974         $d->{netout} += $netdev->{$dev}->{receive};
2975         $d->{netin} += $netdev->{$dev}->{transmit};
2976
2977         if ($full) {
2978             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2979             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2980         }
2981
2982     }
2983
2984     my $ctime = gettimeofday;
2985
2986     foreach my $vmid (keys %$list) {
2987
2988         my $d = $res->{$vmid};
2989         my $pid = $d->{pid};
2990         next if !$pid;
2991
2992         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2993         next if !$pstat; # not running
2994
2995         my $used = $pstat->{utime} + $pstat->{stime};
2996
2997         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2998
2999         if ($pstat->{vsize}) {
3000             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3001         }
3002
3003         my $old = $last_proc_pid_stat->{$pid};
3004         if (!$old) {
3005             $last_proc_pid_stat->{$pid} = {
3006                 time => $ctime,
3007                 used => $used,
3008                 cpu => 0,
3009             };
3010             next;
3011         }
3012
3013         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3014
3015         if ($dtime > 1000) {
3016             my $dutime = $used -  $old->{used};
3017
3018             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3019             $last_proc_pid_stat->{$pid} = {
3020                 time => $ctime,
3021                 used => $used,
3022                 cpu => $d->{cpu},
3023             };
3024         } else {
3025             $d->{cpu} = $old->{cpu};
3026         }
3027     }
3028
3029     return $res if !$full;
3030
3031     my $qmpclient = PVE::QMPClient->new();
3032
3033     my $ballooncb = sub {
3034         my ($vmid, $resp) = @_;
3035
3036         my $info = $resp->{'return'};
3037         return if !$info->{max_mem};
3038
3039         my $d = $res->{$vmid};
3040
3041         # use memory assigned to VM
3042         $d->{maxmem} = $info->{max_mem};
3043         $d->{balloon} = $info->{actual};
3044
3045         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3046             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3047             $d->{freemem} = $info->{free_mem};
3048         }
3049
3050         $d->{ballooninfo} = $info;
3051     };
3052
3053     my $blockstatscb = sub {
3054         my ($vmid, $resp) = @_;
3055         my $data = $resp->{'return'} || [];
3056         my $totalrdbytes = 0;
3057         my $totalwrbytes = 0;
3058
3059         for my $blockstat (@$data) {
3060             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3061             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3062
3063             $blockstat->{device} =~ s/drive-//;
3064             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3065         }
3066         $res->{$vmid}->{diskread} = $totalrdbytes;
3067         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3068     };
3069
3070     my $machinecb = sub {
3071         my ($vmid, $resp) = @_;
3072         my $data = $resp->{'return'} || [];
3073
3074         $res->{$vmid}->{'running-machine'} =
3075             PVE::QemuServer::Machine::current_from_query_machines($data);
3076     };
3077
3078     my $versioncb = sub {
3079         my ($vmid, $resp) = @_;
3080         my $data = $resp->{'return'} // {};
3081         my $version = 'unknown';
3082
3083         if (my $v = $data->{qemu}) {
3084             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3085         }
3086
3087         $res->{$vmid}->{'running-qemu'} = $version;
3088     };
3089
3090     my $statuscb = sub {
3091         my ($vmid, $resp) = @_;
3092
3093         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3094         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3095         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3096         # this fails if ballon driver is not loaded, so this must be
3097         # the last commnand (following command are aborted if this fails).
3098         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3099
3100         my $status = 'unknown';
3101         if (!defined($status = $resp->{'return'}->{status})) {
3102             warn "unable to get VM status\n";
3103             return;
3104         }
3105
3106         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3107     };
3108
3109     foreach my $vmid (keys %$list) {
3110         next if $opt_vmid && ($vmid ne $opt_vmid);
3111         next if !$res->{$vmid}->{pid}; # not running
3112         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3113     }
3114
3115     $qmpclient->queue_execute(undef, 2);
3116
3117     foreach my $vmid (keys %$list) {
3118         next if $opt_vmid && ($vmid ne $opt_vmid);
3119         next if !$res->{$vmid}->{pid}; #not running
3120
3121         # we can't use the $qmpclient since it might have already aborted on
3122         # 'query-balloon', but this might also fail for older versions...
3123         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3124         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3125     }
3126
3127     foreach my $vmid (keys %$list) {
3128         next if $opt_vmid && ($vmid ne $opt_vmid);
3129         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3130     }
3131
3132     return $res;
3133 }
3134
3135 sub conf_has_serial {
3136     my ($conf) = @_;
3137
3138     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3139         if ($conf->{"serial$i"}) {
3140             return 1;
3141         }
3142     }
3143
3144     return 0;
3145 }
3146
3147 sub conf_has_audio {
3148     my ($conf, $id) = @_;
3149
3150     $id //= 0;
3151     my $audio = $conf->{"audio$id"};
3152     return if !defined($audio);
3153
3154     my $audioproperties = parse_property_string($audio_fmt, $audio);
3155     my $audiodriver = $audioproperties->{driver} // 'spice';
3156
3157     return {
3158         dev => $audioproperties->{device},
3159         dev_id => "audiodev$id",
3160         backend => $audiodriver,
3161         backend_id => "$audiodriver-backend${id}",
3162     };
3163 }
3164
3165 sub audio_devs {
3166     my ($audio, $audiopciaddr, $machine_version) = @_;
3167
3168     my $devs = [];
3169
3170     my $id = $audio->{dev_id};
3171     my $audiodev = "";
3172     if (min_version($machine_version, 4, 2)) {
3173         $audiodev = ",audiodev=$audio->{backend_id}";
3174     }
3175
3176     if ($audio->{dev} eq 'AC97') {
3177         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3178     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3179         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3180         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3181         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3182     } else {
3183         die "unkown audio device '$audio->{dev}', implement me!";
3184     }
3185
3186     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3187
3188     return $devs;
3189 }
3190
3191 sub get_tpm_paths {
3192     my ($vmid) = @_;
3193     return {
3194         socket => "/var/run/qemu-server/$vmid.swtpm",
3195         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3196     };
3197 }
3198
3199 sub add_tpm_device {
3200     my ($vmid, $devices, $conf) = @_;
3201
3202     return if !$conf->{tpmstate0};
3203
3204     my $paths = get_tpm_paths($vmid);
3205
3206     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3207     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3208     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3209 }
3210
3211 sub start_swtpm {
3212     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3213
3214     return if !$tpmdrive;
3215
3216     my $state;
3217     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3218     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3219     if ($storeid) {
3220         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3221     } else {
3222         $state = $tpm->{file};
3223     }
3224
3225     my $paths = get_tpm_paths($vmid);
3226
3227     # during migration, we will get state from remote
3228     #
3229     if (!$migration) {
3230         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3231         my $setup_cmd = [
3232             "swtpm_setup",
3233             "--tpmstate",
3234             "file://$state",
3235             "--createek",
3236             "--create-ek-cert",
3237             "--create-platform-cert",
3238             "--lock-nvram",
3239             "--config",
3240             "/etc/swtpm_setup.conf", # do not use XDG configs
3241             "--runas",
3242             "0", # force creation as root, error if not possible
3243             "--not-overwrite", # ignore existing state, do not modify
3244         ];
3245
3246         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3247         # TPM 2.0 supports ECC crypto, use if possible
3248         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3249
3250         run_command($setup_cmd, outfunc => sub {
3251             print "swtpm_setup: $1\n";
3252         });
3253     }
3254
3255     my $emulator_cmd = [
3256         "swtpm",
3257         "socket",
3258         "--tpmstate",
3259         "backend-uri=file://$state,mode=0600",
3260         "--ctrl",
3261         "type=unixio,path=$paths->{socket},mode=0600",
3262         "--pid",
3263         "file=$paths->{pid}",
3264         "--terminate", # terminate on QEMU disconnect
3265         "--daemon",
3266     ];
3267     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3268     run_command($emulator_cmd, outfunc => sub { print $1; });
3269
3270     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3271     while (! -e $paths->{pid}) {
3272         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3273         usleep(50_000);
3274     }
3275
3276     # return untainted PID of swtpm daemon so it can be killed on error
3277     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3278     return $1;
3279 }
3280
3281 sub vga_conf_has_spice {
3282     my ($vga) = @_;
3283
3284     my $vgaconf = parse_vga($vga);
3285     my $vgatype = $vgaconf->{type};
3286     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3287
3288     return $1 || 1;
3289 }
3290
3291 sub is_native($) {
3292     my ($arch) = @_;
3293     return get_host_arch() eq $arch;
3294 }
3295
3296 sub get_vm_arch {
3297     my ($conf) = @_;
3298     return $conf->{arch} // get_host_arch();
3299 }
3300
3301 my $default_machines = {
3302     x86_64 => 'pc',
3303     aarch64 => 'virt',
3304 };
3305
3306 sub get_installed_machine_version {
3307     my ($kvmversion) = @_;
3308     $kvmversion = kvm_user_version() if !defined($kvmversion);
3309     $kvmversion =~ m/^(\d+\.\d+)/;
3310     return $1;
3311 }
3312
3313 sub windows_get_pinned_machine_version {
3314     my ($machine, $base_version, $kvmversion) = @_;
3315
3316     my $pin_version = $base_version;
3317     if (!defined($base_version) ||
3318         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3319     ) {
3320         $pin_version = get_installed_machine_version($kvmversion);
3321     }
3322     if (!$machine || $machine eq 'pc') {
3323         $machine = "pc-i440fx-$pin_version";
3324     } elsif ($machine eq 'q35') {
3325         $machine = "pc-q35-$pin_version";
3326     } elsif ($machine eq 'virt') {
3327         $machine = "virt-$pin_version";
3328     } else {
3329         warn "unknown machine type '$machine', not touching that!\n";
3330     }
3331
3332     return $machine;
3333 }
3334
3335 sub get_vm_machine {
3336     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3337
3338     my $machine = $forcemachine || $conf->{machine};
3339
3340     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3341         $kvmversion //= kvm_user_version();
3342         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3343         # layout which confuses windows quite a bit and may result in various regressions..
3344         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3345         if (windows_version($conf->{ostype})) {
3346             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3347         }
3348         $arch //= 'x86_64';
3349         $machine ||= $default_machines->{$arch};
3350         if ($add_pve_version) {
3351             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3352             $machine .= "+pve$pvever";
3353         }
3354     }
3355
3356     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3357         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3358         $machine = $1 if $is_pxe;
3359
3360         # for version-pinned machines that do not include a pve-version (e.g.
3361         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3362         $machine .= '+pve0';
3363
3364         $machine .= '.pxe' if $is_pxe;
3365     }
3366
3367     return $machine;
3368 }
3369
3370 sub get_ovmf_files($$$) {
3371     my ($arch, $efidisk, $smm) = @_;
3372
3373     my $types = $OVMF->{$arch}
3374         or die "no OVMF images known for architecture '$arch'\n";
3375
3376     my $type = 'default';
3377     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3378         $type = $smm ? "4m" : "4m-no-smm";
3379         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3380     }
3381
3382     return $types->{$type}->@*;
3383 }
3384
3385 my $Arch2Qemu = {
3386     aarch64 => '/usr/bin/qemu-system-aarch64',
3387     x86_64 => '/usr/bin/qemu-system-x86_64',
3388 };
3389 sub get_command_for_arch($) {
3390     my ($arch) = @_;
3391     return '/usr/bin/kvm' if is_native($arch);
3392
3393     my $cmd = $Arch2Qemu->{$arch}
3394         or die "don't know how to emulate architecture '$arch'\n";
3395     return $cmd;
3396 }
3397
3398 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3399 # to use in a QEMU command line (-cpu element), first array_intersect the result
3400 # of query_supported_ with query_understood_. This is necessary because:
3401 #
3402 # a) query_understood_ returns flags the host cannot use and
3403 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3404 #    flags, but CPU settings - with most of them being flags. Those settings
3405 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3406 #
3407 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3408 # expensive. If you need the value returned from this, you can get it much
3409 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3410 # $accel being 'kvm' or 'tcg'.
3411 #
3412 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3413 # changes, automatically populating pmxcfs.
3414 #
3415 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3416 # since kvm and tcg machines support different flags
3417 #
3418 sub query_supported_cpu_flags {
3419     my ($arch) = @_;
3420
3421     $arch //= get_host_arch();
3422     my $default_machine = $default_machines->{$arch};
3423
3424     my $flags = {};
3425
3426     # FIXME: Once this is merged, the code below should work for ARM as well:
3427     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3428     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3429         $arch eq "aarch64";
3430
3431     my $kvm_supported = defined(kvm_version());
3432     my $qemu_cmd = get_command_for_arch($arch);
3433     my $fakevmid = -1;
3434     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3435
3436     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3437     my $query_supported_run_qemu = sub {
3438         my ($kvm) = @_;
3439
3440         my $flags = {};
3441         my $cmd = [
3442             $qemu_cmd,
3443             '-machine', $default_machine,
3444             '-display', 'none',
3445             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3446             '-mon', 'chardev=qmp,mode=control',
3447             '-pidfile', $pidfile,
3448             '-S', '-daemonize'
3449         ];
3450
3451         if (!$kvm) {
3452             push @$cmd, '-accel', 'tcg';
3453         }
3454
3455         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3456         die "QEMU flag querying VM exited with code " . $rc if $rc;
3457
3458         eval {
3459             my $cmd_result = mon_cmd(
3460                 $fakevmid,
3461                 'query-cpu-model-expansion',
3462                 type => 'full',
3463                 model => { name => 'host' }
3464             );
3465
3466             my $props = $cmd_result->{model}->{props};
3467             foreach my $prop (keys %$props) {
3468                 next if $props->{$prop} ne '1';
3469                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3470                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3471                 # We only keep those with underscores, to match /proc/cpuinfo
3472                 $prop =~ s/\.|-/_/g;
3473                 $flags->{$prop} = 1;
3474             }
3475         };
3476         my $err = $@;
3477
3478         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3479         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3480
3481         die $err if $err;
3482
3483         return [ sort keys %$flags ];
3484     };
3485
3486     # We need to query QEMU twice, since KVM and TCG have different supported flags
3487     PVE::QemuConfig->lock_config($fakevmid, sub {
3488         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3489         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3490
3491         if ($kvm_supported) {
3492             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3493             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3494         }
3495     });
3496
3497     return $flags;
3498 }
3499
3500 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3501 my $understood_cpu_flag_dir = "/usr/share/kvm";
3502 sub query_understood_cpu_flags {
3503     my $arch = get_host_arch();
3504     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3505
3506     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3507         if ! -e $filepath;
3508
3509     my $raw = file_get_contents($filepath);
3510     $raw =~ s/^\s+|\s+$//g;
3511     my @flags = split(/\s+/, $raw);
3512
3513     return \@flags;
3514 }
3515
3516 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3517 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3518 my sub should_disable_smm {
3519     my ($conf, $vga) = @_;
3520
3521     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3522         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3523 }
3524
3525 my sub print_ovmf_drive_commandlines {
3526     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3527
3528     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3529
3530     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3531     die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3532
3533     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3534     if ($d) {
3535         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3536         my ($path, $format) = $d->@{'file', 'format'};
3537         if ($storeid) {
3538             $path = PVE::Storage::path($storecfg, $d->{file});
3539             if (!defined($format)) {
3540                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3541                 $format = qemu_img_format($scfg, $volname);
3542             }
3543         } elsif (!defined($format)) {
3544             die "efidisk format must be specified\n";
3545         }
3546         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3547         if ($path =~ m/^rbd:/) {
3548             $var_drive_str .= ',cache=writeback';
3549             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3550         }
3551         $var_drive_str .= ",format=$format,file=$path";
3552
3553         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3554         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3555     } else {
3556         log_warn("no efidisk configured! Using temporary efivars disk.");
3557         my $path = "/tmp/$vmid-ovmf.fd";
3558         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3559         $var_drive_str .= ",format=raw,file=$path";
3560         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3561     }
3562
3563     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3564 }
3565
3566 sub config_to_command {
3567     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3568         $pbs_backing) = @_;
3569
3570     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3571     my $devices = [];
3572     my $bridges = {};
3573     my $ostype = $conf->{ostype};
3574     my $winversion = windows_version($ostype);
3575     my $kvm = $conf->{kvm};
3576     my $nodename = nodename();
3577
3578     my $arch = get_vm_arch($conf);
3579     my $kvm_binary = get_command_for_arch($arch);
3580     my $kvmver = kvm_user_version($kvm_binary);
3581
3582     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3583         $kvmver //= "undefined";
3584         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3585     }
3586
3587     my $add_pve_version = min_version($kvmver, 4, 1);
3588
3589     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3590     my $machine_version = extract_version($machine_type, $kvmver);
3591     $kvm //= 1 if is_native($arch);
3592
3593     $machine_version =~ m/(\d+)\.(\d+)/;
3594     my ($machine_major, $machine_minor) = ($1, $2);
3595
3596     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3597         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3598     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3599         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3600             ." please upgrade node '$nodename'\n"
3601     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3602         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3603         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3604             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3605             ." node '$nodename'\n";
3606     }
3607
3608     # if a specific +pve version is required for a feature, use $version_guard
3609     # instead of min_version to allow machines to be run with the minimum
3610     # required version
3611     my $required_pve_version = 0;
3612     my $version_guard = sub {
3613         my ($major, $minor, $pve) = @_;
3614         return 0 if !min_version($machine_version, $major, $minor, $pve);
3615         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3616         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3617         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3618         return 1;
3619     };
3620
3621     if ($kvm && !defined kvm_version()) {
3622         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3623             ." or enable in BIOS.\n";
3624     }
3625
3626     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3627     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3628     my $use_old_bios_files = undef;
3629     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3630
3631     my $cmd = [];
3632     if ($conf->{affinity}) {
3633         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3634     }
3635
3636     push @$cmd, $kvm_binary;
3637
3638     push @$cmd, '-id', $vmid;
3639
3640     my $vmname = $conf->{name} || "vm$vmid";
3641
3642     push @$cmd, '-name', "$vmname,debug-threads=on";
3643
3644     push @$cmd, '-no-shutdown';
3645
3646     my $use_virtio = 0;
3647
3648     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3649     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3650     push @$cmd, '-mon', "chardev=qmp,mode=control";
3651
3652     if (min_version($machine_version, 2, 12)) {
3653         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3654         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3655     }
3656
3657     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3658
3659     push @$cmd, '-daemonize';
3660
3661     if ($conf->{smbios1}) {
3662         my $smbios_conf = parse_smbios1($conf->{smbios1});
3663         if ($smbios_conf->{base64}) {
3664             # Do not pass base64 flag to qemu
3665             delete $smbios_conf->{base64};
3666             my $smbios_string = "";
3667             foreach my $key (keys %$smbios_conf) {
3668                 my $value;
3669                 if ($key eq "uuid") {
3670                     $value = $smbios_conf->{uuid}
3671                 } else {
3672                     $value = decode_base64($smbios_conf->{$key});
3673                 }
3674                 # qemu accepts any binary data, only commas need escaping by double comma
3675                 $value =~ s/,/,,/g;
3676                 $smbios_string .= "," . $key . "=" . $value if $value;
3677             }
3678             push @$cmd, '-smbios', "type=1" . $smbios_string;
3679         } else {
3680             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3681         }
3682     }
3683
3684     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3685         my ($code_drive_str, $var_drive_str) =
3686             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3687         push $cmd->@*, '-drive', $code_drive_str;
3688         push $cmd->@*, '-drive', $var_drive_str;
3689     }
3690
3691     if ($q35) { # tell QEMU to load q35 config early
3692         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3693         if (min_version($machine_version, 4, 0)) {
3694             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3695         } else {
3696             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3697         }
3698     }
3699
3700     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3701         push @$cmd, $fixups->@*;
3702     }
3703
3704     if ($conf->{vmgenid}) {
3705         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3706     }
3707
3708     # add usb controllers
3709     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3710         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3711     push @$devices, @usbcontrollers if @usbcontrollers;
3712     my $vga = parse_vga($conf->{vga});
3713
3714     my $qxlnum = vga_conf_has_spice($conf->{vga});
3715     $vga->{type} = 'qxl' if $qxlnum;
3716
3717     if (!$vga->{type}) {
3718         if ($arch eq 'aarch64') {
3719             $vga->{type} = 'virtio';
3720         } elsif (min_version($machine_version, 2, 9)) {
3721             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3722         } else {
3723             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3724         }
3725     }
3726
3727     # enable absolute mouse coordinates (needed by vnc)
3728     my $tablet = $conf->{tablet};
3729     if (!defined($tablet)) {
3730         $tablet = $defaults->{tablet};
3731         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3732         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3733     }
3734
3735     if ($tablet) {
3736         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3737         my $kbd = print_keyboarddevice_full($conf, $arch);
3738         push @$devices, '-device', $kbd if defined($kbd);
3739     }
3740
3741     my $bootorder = device_bootorder($conf);
3742
3743     # host pci device passthrough
3744     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3745         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3746
3747     # usb devices
3748     my $usb_dev_features = {};
3749     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3750
3751     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3752         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3753     push @$devices, @usbdevices if @usbdevices;
3754
3755     # serial devices
3756     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3757         my $path = $conf->{"serial$i"} or next;
3758         if ($path eq 'socket') {
3759             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3760             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3761             # On aarch64, serial0 is the UART device. QEMU only allows
3762             # connecting UART devices via the '-serial' command line, as
3763             # the device has a fixed slot on the hardware...
3764             if ($arch eq 'aarch64' && $i == 0) {
3765                 push @$devices, '-serial', "chardev:serial$i";
3766             } else {
3767                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3768             }
3769         } else {
3770             die "no such serial device\n" if ! -c $path;
3771             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3772             push @$devices, '-device', "isa-serial,chardev=serial$i";
3773         }
3774     }
3775
3776     # parallel devices
3777     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3778         if (my $path = $conf->{"parallel$i"}) {
3779             die "no such parallel device\n" if ! -c $path;
3780             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3781             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3782             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3783         }
3784     }
3785
3786     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3787         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3788         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3789         push @$devices, @$audio_devs;
3790     }
3791
3792     add_tpm_device($vmid, $devices, $conf);
3793
3794     my $sockets = 1;
3795     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3796     $sockets = $conf->{sockets} if  $conf->{sockets};
3797
3798     my $cores = $conf->{cores} || 1;
3799
3800     my $maxcpus = $sockets * $cores;
3801
3802     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3803
3804     my $allowed_vcpus = $cpuinfo->{cpus};
3805
3806     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3807
3808     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3809         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3810         for (my $i = 2; $i <= $vcpus; $i++)  {
3811             my $cpustr = print_cpu_device($conf,$i);
3812             push @$cmd, '-device', $cpustr;
3813         }
3814
3815     } else {
3816
3817         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3818     }
3819     push @$cmd, '-nodefaults';
3820
3821     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3822
3823     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3824
3825     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3826
3827     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3828         push @$devices, '-device', print_vga_device(
3829             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3830
3831         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3832
3833         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3834         push @$cmd,  '-vnc', "unix:$socket,password=on";
3835     } else {
3836         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3837         push @$cmd, '-nographic';
3838     }
3839
3840     # time drift fix
3841     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3842     my $useLocaltime = $conf->{localtime};
3843
3844     if ($winversion >= 5) { # windows
3845         $useLocaltime = 1 if !defined($conf->{localtime});
3846
3847         # use time drift fix when acpi is enabled
3848         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3849             $tdf = 1 if !defined($conf->{tdf});
3850         }
3851     }
3852
3853     if ($winversion >= 6) {
3854         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3855         push @$cmd, '-no-hpet';
3856     }
3857
3858     push @$rtcFlags, 'driftfix=slew' if $tdf;
3859
3860     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3861         push @$rtcFlags, "base=$conf->{startdate}";
3862     } elsif ($useLocaltime) {
3863         push @$rtcFlags, 'base=localtime';
3864     }
3865
3866     if ($forcecpu) {
3867         push @$cmd, '-cpu', $forcecpu;
3868     } else {
3869         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3870     }
3871
3872     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3873
3874     push @$cmd, '-S' if $conf->{freeze};
3875
3876     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3877
3878     my $guest_agent = parse_guest_agent($conf);
3879
3880     if ($guest_agent->{enabled}) {
3881         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3882         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3883
3884         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3885             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3886             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3887             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3888         } elsif ($guest_agent->{type} eq 'isa') {
3889             push @$devices, '-device', "isa-serial,chardev=qga0";
3890         }
3891     }
3892
3893     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3894     if ($rng && $version_guard->(4, 1, 2)) {
3895         check_rng_source($rng->{source});
3896
3897         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3898         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3899         my $limiter_str = "";
3900         if ($max_bytes) {
3901             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3902         }
3903
3904         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3905         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3906         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3907     }
3908
3909     my $spice_port;
3910
3911     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3912         if ($qxlnum > 1) {
3913             if ($winversion){
3914                 for (my $i = 1; $i < $qxlnum; $i++){
3915                     push @$devices, '-device', print_vga_device(
3916                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3917                 }
3918             } else {
3919                 # assume other OS works like Linux
3920                 my ($ram, $vram) = ("134217728", "67108864");
3921                 if ($vga->{memory}) {
3922                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3923                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3924                 }
3925                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3926                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3927             }
3928         }
3929
3930         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3931
3932         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3933         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3934         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3935
3936         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3937         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3938         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3939
3940         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3941         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3942
3943         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3944         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3945         if ($spice_enhancement->{foldersharing}) {
3946             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3947             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3948         }
3949
3950         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3951         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3952             if $spice_enhancement->{videostreaming};
3953
3954         push @$devices, '-spice', "$spice_opts";
3955     }
3956
3957     # enable balloon by default, unless explicitly disabled
3958     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3959         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3960         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3961         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3962         push @$devices, '-device', $ballooncmd;
3963     }
3964
3965     if ($conf->{watchdog}) {
3966         my $wdopts = parse_watchdog($conf->{watchdog});
3967         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3968         my $watchdog = $wdopts->{model} || 'i6300esb';
3969         push @$devices, '-device', "$watchdog$pciaddr";
3970         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3971     }
3972
3973     my $vollist = [];
3974     my $scsicontroller = {};
3975     my $ahcicontroller = {};
3976     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3977
3978     # Add iscsi initiator name if available
3979     if (my $initiator = get_initiator_name()) {
3980         push @$devices, '-iscsi', "initiator-name=$initiator";
3981     }
3982
3983     PVE::QemuConfig->foreach_volume($conf, sub {
3984         my ($ds, $drive) = @_;
3985
3986         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3987             check_volume_storage_type($storecfg, $drive->{file});
3988             push @$vollist, $drive->{file};
3989         }
3990
3991         # ignore efidisk here, already added in bios/fw handling code above
3992         return if $drive->{interface} eq 'efidisk';
3993         # similar for TPM
3994         return if $drive->{interface} eq 'tpmstate';
3995
3996         $use_virtio = 1 if $ds =~ m/^virtio/;
3997
3998         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3999
4000         if ($drive->{interface} eq 'virtio'){
4001            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4002         }
4003
4004         if ($drive->{interface} eq 'scsi') {
4005
4006             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4007
4008             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4009                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4010
4011             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4012             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4013
4014             my $iothread = '';
4015             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4016                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4017                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4018             } elsif ($drive->{iothread}) {
4019                 log_warn(
4020                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4021                 );
4022             }
4023
4024             my $queues = '';
4025             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4026                 $queues = ",num_queues=$drive->{queues}";
4027             }
4028
4029             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4030                 if !$scsicontroller->{$controller};
4031             $scsicontroller->{$controller}=1;
4032         }
4033
4034         if ($drive->{interface} eq 'sata') {
4035             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4036             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4037             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4038                 if !$ahcicontroller->{$controller};
4039             $ahcicontroller->{$controller}=1;
4040         }
4041
4042         my $pbs_conf = $pbs_backing->{$ds};
4043         my $pbs_name = undef;
4044         if ($pbs_conf) {
4045             $pbs_name = "drive-$ds-pbs";
4046             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4047         }
4048
4049         my $drive_cmd = print_drive_commandline_full(
4050             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4051
4052         # extra protection for templates, but SATA and IDE don't support it..
4053         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4054
4055         push @$devices, '-drive',$drive_cmd;
4056         push @$devices, '-device', print_drivedevice_full(
4057             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4058     });
4059
4060     for (my $i = 0; $i < $MAX_NETS; $i++) {
4061         my $netname = "net$i";
4062
4063         next if !$conf->{$netname};
4064         my $d = parse_net($conf->{$netname});
4065         next if !$d;
4066         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4067
4068         $use_virtio = 1 if $d->{model} eq 'virtio';
4069
4070         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4071
4072         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4073         push @$devices, '-netdev', $netdevfull;
4074
4075         my $netdevicefull = print_netdevice_full(
4076             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4077
4078         push @$devices, '-device', $netdevicefull;
4079     }
4080
4081     if ($conf->{ivshmem}) {
4082         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4083
4084         my $bus;
4085         if ($q35) {
4086             $bus = print_pcie_addr("ivshmem");
4087         } else {
4088             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4089         }
4090
4091         my $ivshmem_name = $ivshmem->{name} // $vmid;
4092         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4093
4094         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4095         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4096             .",size=$ivshmem->{size}M";
4097     }
4098
4099     # pci.4 is nested in pci.1
4100     $bridges->{1} = 1 if $bridges->{4};
4101
4102     if (!$q35) { # add pci bridges
4103         if (min_version($machine_version, 2, 3)) {
4104            $bridges->{1} = 1;
4105            $bridges->{2} = 1;
4106         }
4107         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4108     }
4109
4110     for my $k (sort {$b cmp $a} keys %$bridges) {
4111         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4112
4113         my $k_name = $k;
4114         if ($k == 2 && $legacy_igd) {
4115             $k_name = "$k-igd";
4116         }
4117         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4118         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4119
4120         if ($q35) { # add after -readconfig pve-q35.cfg
4121             splice @$devices, 2, 0, '-device', $devstr;
4122         } else {
4123             unshift @$devices, '-device', $devstr if $k > 0;
4124         }
4125     }
4126
4127     if (!$kvm) {
4128         push @$machineFlags, 'accel=tcg';
4129     }
4130
4131     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4132
4133     my $machine_type_min = $machine_type;
4134     if ($add_pve_version) {
4135         $machine_type_min =~ s/\+pve\d+$//;
4136         $machine_type_min .= "+pve$required_pve_version";
4137     }
4138     push @$machineFlags, "type=${machine_type_min}";
4139
4140     push @$cmd, @$devices;
4141     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4142     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4143     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4144
4145     if (my $vmstate = $conf->{vmstate}) {
4146         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4147         push @$vollist, $vmstate;
4148         push @$cmd, '-loadstate', $statepath;
4149         print "activating and using '$vmstate' as vmstate\n";
4150     }
4151
4152     if (PVE::QemuConfig->is_template($conf)) {
4153         # needed to workaround base volumes being read-only
4154         push @$cmd, '-snapshot';
4155     }
4156
4157     # add custom args
4158     if ($conf->{args}) {
4159         my $aa = PVE::Tools::split_args($conf->{args});
4160         push @$cmd, @$aa;
4161     }
4162
4163     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4164 }
4165
4166 sub check_rng_source {
4167     my ($source) = @_;
4168
4169     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4170     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4171         if ! -e $source;
4172
4173     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4174     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4175         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4176         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4177         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4178             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4179             ." to the host.\n";
4180     }
4181 }
4182
4183 sub spice_port {
4184     my ($vmid) = @_;
4185
4186     my $res = mon_cmd($vmid, 'query-spice');
4187
4188     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4189 }
4190
4191 sub vm_devices_list {
4192     my ($vmid) = @_;
4193
4194     my $res = mon_cmd($vmid, 'query-pci');
4195     my $devices_to_check = [];
4196     my $devices = {};
4197     foreach my $pcibus (@$res) {
4198         push @$devices_to_check, @{$pcibus->{devices}},
4199     }
4200
4201     while (@$devices_to_check) {
4202         my $to_check = [];
4203         for my $d (@$devices_to_check) {
4204             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4205             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4206
4207             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4208             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4209         }
4210         $devices_to_check = $to_check;
4211     }
4212
4213     my $resblock = mon_cmd($vmid, 'query-block');
4214     foreach my $block (@$resblock) {
4215         if($block->{device} =~ m/^drive-(\S+)/){
4216                 $devices->{$1} = 1;
4217         }
4218     }
4219
4220     my $resmice = mon_cmd($vmid, 'query-mice');
4221     foreach my $mice (@$resmice) {
4222         if ($mice->{name} eq 'QEMU HID Tablet') {
4223             $devices->{tablet} = 1;
4224             last;
4225         }
4226     }
4227
4228     # for usb devices there is no query-usb
4229     # but we can iterate over the entries in
4230     # qom-list path=/machine/peripheral
4231     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4232     foreach my $per (@$resperipheral) {
4233         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4234             $devices->{$per->{name}} = 1;
4235         }
4236     }
4237
4238     return $devices;
4239 }
4240
4241 sub vm_deviceplug {
4242     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4243
4244     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4245
4246     my $devices_list = vm_devices_list($vmid);
4247     return 1 if defined($devices_list->{$deviceid});
4248
4249     # add PCI bridge if we need it for the device
4250     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4251
4252     if ($deviceid eq 'tablet') {
4253         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4254     } elsif ($deviceid eq 'keyboard') {
4255         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4256     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4257         my $id = $1;
4258         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4259         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4260     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4261         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4262     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4263         qemu_iothread_add($vmid, $deviceid, $device);
4264
4265         qemu_driveadd($storecfg, $vmid, $device);
4266         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4267
4268         qemu_deviceadd($vmid, $devicefull);
4269         eval { qemu_deviceaddverify($vmid, $deviceid); };
4270         if (my $err = $@) {
4271             eval { qemu_drivedel($vmid, $deviceid); };
4272             warn $@ if $@;
4273             die $err;
4274         }
4275     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4276         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4277         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4278         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4279
4280         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4281
4282         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4283             qemu_iothread_add($vmid, $deviceid, $device);
4284             $devicefull .= ",iothread=iothread-$deviceid";
4285         }
4286
4287         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4288             $devicefull .= ",num_queues=$device->{queues}";
4289         }
4290
4291         qemu_deviceadd($vmid, $devicefull);
4292         qemu_deviceaddverify($vmid, $deviceid);
4293     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4294         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4295         qemu_driveadd($storecfg, $vmid, $device);
4296
4297         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4298         eval { qemu_deviceadd($vmid, $devicefull); };
4299         if (my $err = $@) {
4300             eval { qemu_drivedel($vmid, $deviceid); };
4301             warn $@ if $@;
4302             die $err;
4303         }
4304     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4305         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4306
4307         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4308         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4309         my $use_old_bios_files = undef;
4310         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4311
4312         my $netdevicefull = print_netdevice_full(
4313             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4314         qemu_deviceadd($vmid, $netdevicefull);
4315         eval {
4316             qemu_deviceaddverify($vmid, $deviceid);
4317             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4318         };
4319         if (my $err = $@) {
4320             eval { qemu_netdevdel($vmid, $deviceid); };
4321             warn $@ if $@;
4322             die $err;
4323         }
4324     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4325         my $bridgeid = $2;
4326         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4327         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4328
4329         qemu_deviceadd($vmid, $devicefull);
4330         qemu_deviceaddverify($vmid, $deviceid);
4331     } else {
4332         die "can't hotplug device '$deviceid'\n";
4333     }
4334
4335     return 1;
4336 }
4337
4338 # fixme: this should raise exceptions on error!
4339 sub vm_deviceunplug {
4340     my ($vmid, $conf, $deviceid) = @_;
4341
4342     my $devices_list = vm_devices_list($vmid);
4343     return 1 if !defined($devices_list->{$deviceid});
4344
4345     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4346     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4347
4348     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4349         qemu_devicedel($vmid, $deviceid);
4350     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4351         qemu_devicedel($vmid, $deviceid);
4352         qemu_devicedelverify($vmid, $deviceid);
4353     } elsif ($deviceid =~ m/^usb\d+$/) {
4354         qemu_devicedel($vmid, $deviceid);
4355         qemu_devicedelverify($vmid, $deviceid);
4356     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4357         my $device = parse_drive($deviceid, $conf->{$deviceid});
4358
4359         qemu_devicedel($vmid, $deviceid);
4360         qemu_devicedelverify($vmid, $deviceid);
4361         qemu_drivedel($vmid, $deviceid);
4362         qemu_iothread_del($vmid, $deviceid, $device);
4363     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4364         qemu_devicedel($vmid, $deviceid);
4365         qemu_devicedelverify($vmid, $deviceid);
4366     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4367         my $device = parse_drive($deviceid, $conf->{$deviceid});
4368
4369         qemu_devicedel($vmid, $deviceid);
4370         qemu_devicedelverify($vmid, $deviceid);
4371         qemu_drivedel($vmid, $deviceid);
4372         qemu_deletescsihw($conf, $vmid, $deviceid);
4373
4374         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4375             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4376     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4377         qemu_devicedel($vmid, $deviceid);
4378         qemu_devicedelverify($vmid, $deviceid);
4379         qemu_netdevdel($vmid, $deviceid);
4380     } else {
4381         die "can't unplug device '$deviceid'\n";
4382     }
4383
4384     return 1;
4385 }
4386
4387 sub qemu_spice_usbredir_chardev_add {
4388     my ($vmid, $id) = @_;
4389
4390     mon_cmd($vmid, "chardev-add" , (
4391         id => $id,
4392         backend => {
4393             type => 'spicevmc',
4394             data => {
4395                 type => "usbredir",
4396             },
4397         },
4398     ));
4399 }
4400
4401 sub qemu_deviceadd {
4402     my ($vmid, $devicefull) = @_;
4403
4404     $devicefull = "driver=".$devicefull;
4405     my %options =  split(/[=,]/, $devicefull);
4406
4407     mon_cmd($vmid, "device_add" , %options);
4408 }
4409
4410 sub qemu_devicedel {
4411     my ($vmid, $deviceid) = @_;
4412
4413     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4414 }
4415
4416 sub qemu_iothread_add {
4417     my ($vmid, $deviceid, $device) = @_;
4418
4419     if ($device->{iothread}) {
4420         my $iothreads = vm_iothreads_list($vmid);
4421         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4422     }
4423 }
4424
4425 sub qemu_iothread_del {
4426     my ($vmid, $deviceid, $device) = @_;
4427
4428     if ($device->{iothread}) {
4429         my $iothreads = vm_iothreads_list($vmid);
4430         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4431     }
4432 }
4433
4434 sub qemu_objectadd {
4435     my ($vmid, $objectid, $qomtype) = @_;
4436
4437     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4438
4439     return 1;
4440 }
4441
4442 sub qemu_objectdel {
4443     my ($vmid, $objectid) = @_;
4444
4445     mon_cmd($vmid, "object-del", id => $objectid);
4446
4447     return 1;
4448 }
4449
4450 sub qemu_driveadd {
4451     my ($storecfg, $vmid, $device) = @_;
4452
4453     my $kvmver = get_running_qemu_version($vmid);
4454     my $io_uring = min_version($kvmver, 6, 0);
4455     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4456     $drive =~ s/\\/\\\\/g;
4457     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4458
4459     # If the command succeeds qemu prints: "OK"
4460     return 1 if $ret =~ m/OK/s;
4461
4462     die "adding drive failed: $ret\n";
4463 }
4464
4465 sub qemu_drivedel {
4466     my ($vmid, $deviceid) = @_;
4467
4468     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4469     $ret =~ s/^\s+//;
4470
4471     return 1 if $ret eq "";
4472
4473     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4474     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4475
4476     die "deleting drive $deviceid failed : $ret\n";
4477 }
4478
4479 sub qemu_deviceaddverify {
4480     my ($vmid, $deviceid) = @_;
4481
4482     for (my $i = 0; $i <= 5; $i++) {
4483          my $devices_list = vm_devices_list($vmid);
4484          return 1 if defined($devices_list->{$deviceid});
4485          sleep 1;
4486     }
4487
4488     die "error on hotplug device '$deviceid'\n";
4489 }
4490
4491
4492 sub qemu_devicedelverify {
4493     my ($vmid, $deviceid) = @_;
4494
4495     # need to verify that the device is correctly removed as device_del
4496     # is async and empty return is not reliable
4497
4498     for (my $i = 0; $i <= 5; $i++) {
4499          my $devices_list = vm_devices_list($vmid);
4500          return 1 if !defined($devices_list->{$deviceid});
4501          sleep 1;
4502     }
4503
4504     die "error on hot-unplugging device '$deviceid'\n";
4505 }
4506
4507 sub qemu_findorcreatescsihw {
4508     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4509
4510     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4511
4512     my $scsihwid="$controller_prefix$controller";
4513     my $devices_list = vm_devices_list($vmid);
4514
4515     if (!defined($devices_list->{$scsihwid})) {
4516         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4517     }
4518
4519     return 1;
4520 }
4521
4522 sub qemu_deletescsihw {
4523     my ($conf, $vmid, $opt) = @_;
4524
4525     my $device = parse_drive($opt, $conf->{$opt});
4526
4527     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4528         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4529         return 1;
4530     }
4531
4532     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4533
4534     my $devices_list = vm_devices_list($vmid);
4535     foreach my $opt (keys %{$devices_list}) {
4536         if (is_valid_drivename($opt)) {
4537             my $drive = parse_drive($opt, $conf->{$opt});
4538             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4539                 return 1;
4540             }
4541         }
4542     }
4543
4544     my $scsihwid="scsihw$controller";
4545
4546     vm_deviceunplug($vmid, $conf, $scsihwid);
4547
4548     return 1;
4549 }
4550
4551 sub qemu_add_pci_bridge {
4552     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4553
4554     my $bridges = {};
4555
4556     my $bridgeid;
4557
4558     print_pci_addr($device, $bridges, $arch, $machine_type);
4559
4560     while (my ($k, $v) = each %$bridges) {
4561         $bridgeid = $k;
4562     }
4563     return 1 if !defined($bridgeid) || $bridgeid < 1;
4564
4565     my $bridge = "pci.$bridgeid";
4566     my $devices_list = vm_devices_list($vmid);
4567
4568     if (!defined($devices_list->{$bridge})) {
4569         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4570     }
4571
4572     return 1;
4573 }
4574
4575 sub qemu_set_link_status {
4576     my ($vmid, $device, $up) = @_;
4577
4578     mon_cmd($vmid, "set_link", name => $device,
4579                up => $up ? JSON::true : JSON::false);
4580 }
4581
4582 sub qemu_netdevadd {
4583     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4584
4585     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4586     my %options =  split(/[=,]/, $netdev);
4587
4588     if (defined(my $vhost = $options{vhost})) {
4589         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4590     }
4591
4592     if (defined(my $queues = $options{queues})) {
4593         $options{queues} = $queues + 0;
4594     }
4595
4596     mon_cmd($vmid, "netdev_add",  %options);
4597     return 1;
4598 }
4599
4600 sub qemu_netdevdel {
4601     my ($vmid, $deviceid) = @_;
4602
4603     mon_cmd($vmid, "netdev_del", id => $deviceid);
4604 }
4605
4606 sub qemu_usb_hotplug {
4607     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4608
4609     return if !$device;
4610
4611     # remove the old one first
4612     vm_deviceunplug($vmid, $conf, $deviceid);
4613
4614     # check if xhci controller is necessary and available
4615     my $devicelist = vm_devices_list($vmid);
4616
4617     if (!$devicelist->{xhci}) {
4618         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4619         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4620     }
4621
4622     # print_usbdevice_full expects the parsed device
4623     my $d = parse_usb_device($device->{host});
4624     $d->{usb3} = $device->{usb3};
4625
4626     # add the new one
4627     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4628 }
4629
4630 sub qemu_cpu_hotplug {
4631     my ($vmid, $conf, $vcpus) = @_;
4632
4633     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4634
4635     my $sockets = 1;
4636     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4637     $sockets = $conf->{sockets} if  $conf->{sockets};
4638     my $cores = $conf->{cores} || 1;
4639     my $maxcpus = $sockets * $cores;
4640
4641     $vcpus = $maxcpus if !$vcpus;
4642
4643     die "you can't add more vcpus than maxcpus\n"
4644         if $vcpus > $maxcpus;
4645
4646     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4647
4648     if ($vcpus < $currentvcpus) {
4649
4650         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4651
4652             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4653                 qemu_devicedel($vmid, "cpu$i");
4654                 my $retry = 0;
4655                 my $currentrunningvcpus = undef;
4656                 while (1) {
4657                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4658                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4659                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4660                     $retry++;
4661                     sleep 1;
4662                 }
4663                 #update conf after each succesfull cpu unplug
4664                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4665                 PVE::QemuConfig->write_config($vmid, $conf);
4666             }
4667         } else {
4668             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4669         }
4670
4671         return;
4672     }
4673
4674     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4675     die "vcpus in running vm does not match its configuration\n"
4676         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4677
4678     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4679
4680         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4681             my $cpustr = print_cpu_device($conf, $i);
4682             qemu_deviceadd($vmid, $cpustr);
4683
4684             my $retry = 0;
4685             my $currentrunningvcpus = undef;
4686             while (1) {
4687                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4688                 last if scalar(@{$currentrunningvcpus}) == $i;
4689                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4690                 sleep 1;
4691                 $retry++;
4692             }
4693             #update conf after each succesfull cpu hotplug
4694             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4695             PVE::QemuConfig->write_config($vmid, $conf);
4696         }
4697     } else {
4698
4699         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4700             mon_cmd($vmid, "cpu-add", id => int($i));
4701         }
4702     }
4703 }
4704
4705 sub qemu_block_set_io_throttle {
4706     my ($vmid, $deviceid,
4707         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4708         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4709         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4710         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4711
4712     return if !check_running($vmid) ;
4713
4714     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4715         bps => int($bps),
4716         bps_rd => int($bps_rd),
4717         bps_wr => int($bps_wr),
4718         iops => int($iops),
4719         iops_rd => int($iops_rd),
4720         iops_wr => int($iops_wr),
4721         bps_max => int($bps_max),
4722         bps_rd_max => int($bps_rd_max),
4723         bps_wr_max => int($bps_wr_max),
4724         iops_max => int($iops_max),
4725         iops_rd_max => int($iops_rd_max),
4726         iops_wr_max => int($iops_wr_max),
4727         bps_max_length => int($bps_max_length),
4728         bps_rd_max_length => int($bps_rd_max_length),
4729         bps_wr_max_length => int($bps_wr_max_length),
4730         iops_max_length => int($iops_max_length),
4731         iops_rd_max_length => int($iops_rd_max_length),
4732         iops_wr_max_length => int($iops_wr_max_length),
4733     );
4734
4735 }
4736
4737 sub qemu_block_resize {
4738     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4739
4740     my $running = check_running($vmid);
4741
4742     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4743
4744     return if !$running;
4745
4746     my $padding = (1024 - $size % 1024) % 1024;
4747     $size = $size + $padding;
4748
4749     mon_cmd(
4750         $vmid,
4751         "block_resize",
4752         device => $deviceid,
4753         size => int($size),
4754         timeout => 60,
4755     );
4756 }
4757
4758 sub qemu_volume_snapshot {
4759     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4760
4761     my $running = check_running($vmid);
4762
4763     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4764         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4765     } else {
4766         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4767     }
4768 }
4769
4770 sub qemu_volume_snapshot_delete {
4771     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4772
4773     my $running = check_running($vmid);
4774
4775     if($running) {
4776
4777         $running = undef;
4778         my $conf = PVE::QemuConfig->load_config($vmid);
4779         PVE::QemuConfig->foreach_volume($conf, sub {
4780             my ($ds, $drive) = @_;
4781             $running = 1 if $drive->{file} eq $volid;
4782         });
4783     }
4784
4785     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4786         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4787     } else {
4788         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4789     }
4790 }
4791
4792 sub set_migration_caps {
4793     my ($vmid, $savevm) = @_;
4794
4795     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4796
4797     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4798     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4799
4800     my $cap_ref = [];
4801
4802     my $enabled_cap = {
4803         "auto-converge" => 1,
4804         "xbzrle" => 1,
4805         "x-rdma-pin-all" => 0,
4806         "zero-blocks" => 0,
4807         "compress" => 0,
4808         "dirty-bitmaps" => $dirty_bitmaps,
4809     };
4810
4811     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4812
4813     for my $supported_capability (@$supported_capabilities) {
4814         push @$cap_ref, {
4815             capability => $supported_capability->{capability},
4816             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4817         };
4818     }
4819
4820     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4821 }
4822
4823 sub foreach_volid {
4824     my ($conf, $func, @param) = @_;
4825
4826     my $volhash = {};
4827
4828     my $test_volid = sub {
4829         my ($key, $drive, $snapname) = @_;
4830
4831         my $volid = $drive->{file};
4832         return if !$volid;
4833
4834         $volhash->{$volid}->{cdrom} //= 1;
4835         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4836
4837         my $replicate = $drive->{replicate} // 1;
4838         $volhash->{$volid}->{replicate} //= 0;
4839         $volhash->{$volid}->{replicate} = 1 if $replicate;
4840
4841         $volhash->{$volid}->{shared} //= 0;
4842         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4843
4844         $volhash->{$volid}->{referenced_in_config} //= 0;
4845         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4846
4847         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4848             if defined($snapname);
4849
4850         my $size = $drive->{size};
4851         $volhash->{$volid}->{size} //= $size if $size;
4852
4853         $volhash->{$volid}->{is_vmstate} //= 0;
4854         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4855
4856         $volhash->{$volid}->{is_tpmstate} //= 0;
4857         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4858
4859         $volhash->{$volid}->{is_unused} //= 0;
4860         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4861
4862         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4863     };
4864
4865     my $include_opts = {
4866         extra_keys => ['vmstate'],
4867         include_unused => 1,
4868     };
4869
4870     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4871     foreach my $snapname (keys %{$conf->{snapshots}}) {
4872         my $snap = $conf->{snapshots}->{$snapname};
4873         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4874     }
4875
4876     foreach my $volid (keys %$volhash) {
4877         &$func($volid, $volhash->{$volid}, @param);
4878     }
4879 }
4880
4881 my $fast_plug_option = {
4882     'lock' => 1,
4883     'name' => 1,
4884     'onboot' => 1,
4885     'shares' => 1,
4886     'startup' => 1,
4887     'description' => 1,
4888     'protection' => 1,
4889     'vmstatestorage' => 1,
4890     'hookscript' => 1,
4891     'tags' => 1,
4892 };
4893
4894 for my $opt (keys %$confdesc_cloudinit) {
4895     $fast_plug_option->{$opt} = 1;
4896 };
4897
4898 # hotplug changes in [PENDING]
4899 # $selection hash can be used to only apply specified options, for
4900 # example: { cores => 1 } (only apply changed 'cores')
4901 # $errors ref is used to return error messages
4902 sub vmconfig_hotplug_pending {
4903     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4904
4905     my $defaults = load_defaults();
4906     my $arch = get_vm_arch($conf);
4907     my $machine_type = get_vm_machine($conf, undef, $arch);
4908
4909     # commit values which do not have any impact on running VM first
4910     # Note: those option cannot raise errors, we we do not care about
4911     # $selection and always apply them.
4912
4913     my $add_error = sub {
4914         my ($opt, $msg) = @_;
4915         $errors->{$opt} = "hotplug problem - $msg";
4916     };
4917
4918     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4919
4920     my $cloudinit_record_changed = sub {
4921         my ($conf, $opt, $old, $new) = @_;
4922         return if !$cloudinit_pending_properties->{$opt};
4923
4924         my $ci = ($conf->{cloudinit} //= {});
4925
4926         my $recorded = $ci->{$opt};
4927         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4928
4929         if (defined($new)) {
4930             if (defined($old)) {
4931                 # an existing value is being modified
4932                 if (defined($recorded)) {
4933                     # the value was already not in sync
4934                     if ($new eq $recorded) {
4935                         # a value is being reverted to the cloud-init state:
4936                         delete $ci->{$opt};
4937                         delete $added{$opt};
4938                     } else {
4939                         # the value was changed multiple times, do nothing
4940                     }
4941                 } elsif ($added{$opt}) {
4942                     # the value had been marked as added and is being changed, do nothing
4943                 } else {
4944                     # the value is new, record it:
4945                     $ci->{$opt} = $old;
4946                 }
4947             } else {
4948                 # a new value is being added
4949                 if (defined($recorded)) {
4950                     # it was already not in sync
4951                     if ($new eq $recorded) {
4952                         # a value is being reverted to the cloud-init state:
4953                         delete $ci->{$opt};
4954                         delete $added{$opt};
4955                     } else {
4956                         # the value had temporarily been removed, do nothing
4957                     }
4958                 } elsif ($added{$opt}) {
4959                     # the value had been marked as added already, do nothing
4960                 } else {
4961                     # the value is new, add it
4962                     $added{$opt} = 1;
4963                 }
4964             }
4965         } elsif (!defined($old)) {
4966             # a non-existent value is being removed? ignore...
4967         } else {
4968             # a value is being deleted
4969             if (defined($recorded)) {
4970                 # a value was already recorded, just keep it
4971             } elsif ($added{$opt}) {
4972                 # the value was marked as added, remove it
4973                 delete $added{$opt};
4974             } else {
4975                 # a previously unrecorded value is being removed, record the old value:
4976                 $ci->{$opt} = $old;
4977             }
4978         }
4979
4980         my $added = join(',', sort keys %added);
4981         $ci->{added} = $added if length($added);
4982     };
4983
4984     my $changes = 0;
4985     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4986         if ($fast_plug_option->{$opt}) {
4987             my $new = delete $conf->{pending}->{$opt};
4988             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4989             $conf->{$opt} = $new;
4990             $changes = 1;
4991         }
4992     }
4993
4994     if ($changes) {
4995         PVE::QemuConfig->write_config($vmid, $conf);
4996     }
4997
4998     my $ostype = $conf->{ostype};
4999     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5000     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5001     my $usb_hotplug = $hotplug_features->{usb}
5002         && min_version($version, 7, 1)
5003         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5004
5005     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5006     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5007
5008     foreach my $opt (sort keys %$pending_delete_hash) {
5009         next if $selection && !$selection->{$opt};
5010         my $force = $pending_delete_hash->{$opt}->{force};
5011         eval {
5012             if ($opt eq 'hotplug') {
5013                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5014             } elsif ($opt eq 'tablet') {
5015                 die "skip\n" if !$hotplug_features->{usb};
5016                 if ($defaults->{tablet}) {
5017                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5018                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5019                         if $arch eq 'aarch64';
5020                 } else {
5021                     vm_deviceunplug($vmid, $conf, 'tablet');
5022                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5023                 }
5024             } elsif ($opt =~ m/^usb(\d+)$/) {
5025                 my $index = $1;
5026                 die "skip\n" if !$usb_hotplug;
5027                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5028                 vm_deviceunplug($vmid, $conf, $opt);
5029             } elsif ($opt eq 'vcpus') {
5030                 die "skip\n" if !$hotplug_features->{cpu};
5031                 qemu_cpu_hotplug($vmid, $conf, undef);
5032             } elsif ($opt eq 'balloon') {
5033                 # enable balloon device is not hotpluggable
5034                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5035                 # here we reset the ballooning value to memory
5036                 my $balloon = $conf->{memory} || $defaults->{memory};
5037                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038             } elsif ($fast_plug_option->{$opt}) {
5039                 # do nothing
5040             } elsif ($opt =~ m/^net(\d+)$/) {
5041                 die "skip\n" if !$hotplug_features->{network};
5042                 vm_deviceunplug($vmid, $conf, $opt);
5043             } elsif (is_valid_drivename($opt)) {
5044                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5045                 vm_deviceunplug($vmid, $conf, $opt);
5046                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5047             } elsif ($opt =~ m/^memory$/) {
5048                 die "skip\n" if !$hotplug_features->{memory};
5049                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
5050             } elsif ($opt eq 'cpuunits') {
5051                 $cgroup->change_cpu_shares(undef);
5052             } elsif ($opt eq 'cpulimit') {
5053                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5054             } else {
5055                 die "skip\n";
5056             }
5057         };
5058         if (my $err = $@) {
5059             &$add_error($opt, $err) if $err ne "skip\n";
5060         } else {
5061             my $old = delete $conf->{$opt};
5062             $cloudinit_record_changed->($conf, $opt, $old, undef);
5063             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5064         }
5065     }
5066
5067     my $cloudinit_opt;
5068     foreach my $opt (keys %{$conf->{pending}}) {
5069         next if $selection && !$selection->{$opt};
5070         my $value = $conf->{pending}->{$opt};
5071         eval {
5072             if ($opt eq 'hotplug') {
5073                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5074             } elsif ($opt eq 'tablet') {
5075                 die "skip\n" if !$hotplug_features->{usb};
5076                 if ($value == 1) {
5077                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5078                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5079                         if $arch eq 'aarch64';
5080                 } elsif ($value == 0) {
5081                     vm_deviceunplug($vmid, $conf, 'tablet');
5082                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5083                 }
5084             } elsif ($opt =~ m/^usb(\d+)$/) {
5085                 my $index = $1;
5086                 die "skip\n" if !$usb_hotplug;
5087                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5088                 my $id = $opt;
5089                 if ($d->{host} eq 'spice')  {
5090                     $id = "usbredirdev$index";
5091                 }
5092                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5093             } elsif ($opt eq 'vcpus') {
5094                 die "skip\n" if !$hotplug_features->{cpu};
5095                 qemu_cpu_hotplug($vmid, $conf, $value);
5096             } elsif ($opt eq 'balloon') {
5097                 # enable/disable balloning device is not hotpluggable
5098                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5099                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5100                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5101
5102                 # allow manual ballooning if shares is set to zero
5103                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5104                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5105                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5106                 }
5107             } elsif ($opt =~ m/^net(\d+)$/) {
5108                 # some changes can be done without hotplug
5109                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5110                                     $vmid, $opt, $value, $arch, $machine_type);
5111             } elsif (is_valid_drivename($opt)) {
5112                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5113                 # some changes can be done without hotplug
5114                 my $drive = parse_drive($opt, $value);
5115                 if (drive_is_cloudinit($drive)) {
5116                     $cloudinit_opt = [$opt, $drive];
5117                     # apply all the other changes first, then generate the cloudinit disk
5118                     die "skip\n";
5119                 }
5120                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5121                                      $vmid, $opt, $value, $arch, $machine_type);
5122             } elsif ($opt =~ m/^memory$/) { #dimms
5123                 die "skip\n" if !$hotplug_features->{memory};
5124                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5125             } elsif ($opt eq 'cpuunits') {
5126                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5127                 $cgroup->change_cpu_shares($new_cpuunits);
5128             } elsif ($opt eq 'cpulimit') {
5129                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5130                 $cgroup->change_cpu_quota($cpulimit, 100000);
5131             } elsif ($opt eq 'agent') {
5132                 vmconfig_update_agent($conf, $opt, $value);
5133             } else {
5134                 die "skip\n";  # skip non-hot-pluggable options
5135             }
5136         };
5137         if (my $err = $@) {
5138             &$add_error($opt, $err) if $err ne "skip\n";
5139         } else {
5140             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5141             $conf->{$opt} = $value;
5142             delete $conf->{pending}->{$opt};
5143         }
5144     }
5145
5146     if (defined($cloudinit_opt)) {
5147         my ($opt, $drive) = @$cloudinit_opt;
5148         my $value = $conf->{pending}->{$opt};
5149         eval {
5150             my $temp = {%$conf, $opt => $value};
5151             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5152             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5153                                  $vmid, $opt, $value, $arch, $machine_type);
5154         };
5155         if (my $err = $@) {
5156             &$add_error($opt, $err) if $err ne "skip\n";
5157         } else {
5158             $conf->{$opt} = $value;
5159             delete $conf->{pending}->{$opt};
5160         }
5161     }
5162
5163     # unplug xhci controller if no usb device is left
5164     if ($usb_hotplug) {
5165         my $has_usb = 0;
5166         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5167             next if !defined($conf->{"usb$i"});
5168             $has_usb = 1;
5169             last;
5170         }
5171         if (!$has_usb) {
5172             vm_deviceunplug($vmid, $conf, 'xhci');
5173         }
5174     }
5175
5176     PVE::QemuConfig->write_config($vmid, $conf);
5177
5178     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5179         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5180     }
5181 }
5182
5183 sub try_deallocate_drive {
5184     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5185
5186     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5187         my $volid = $drive->{file};
5188         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5189             my $sid = PVE::Storage::parse_volume_id($volid);
5190             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5191
5192             # check if the disk is really unused
5193             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5194                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5195             PVE::Storage::vdisk_free($storecfg, $volid);
5196             return 1;
5197         } else {
5198             # If vm is not owner of this disk remove from config
5199             return 1;
5200         }
5201     }
5202
5203     return;
5204 }
5205
5206 sub vmconfig_delete_or_detach_drive {
5207     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5208
5209     my $drive = parse_drive($opt, $conf->{$opt});
5210
5211     my $rpcenv = PVE::RPCEnvironment::get();
5212     my $authuser = $rpcenv->get_user();
5213
5214     if ($force) {
5215         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5216         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5217     } else {
5218         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5219     }
5220 }
5221
5222
5223
5224 sub vmconfig_apply_pending {
5225     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5226
5227     return if !scalar(keys %{$conf->{pending}});
5228
5229     my $add_apply_error = sub {
5230         my ($opt, $msg) = @_;
5231         my $err_msg = "unable to apply pending change $opt : $msg";
5232         $errors->{$opt} = $err_msg;
5233         warn $err_msg;
5234     };
5235
5236     # cold plug
5237
5238     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5239     foreach my $opt (sort keys %$pending_delete_hash) {
5240         my $force = $pending_delete_hash->{$opt}->{force};
5241         eval {
5242             if ($opt =~ m/^unused/) {
5243                 die "internal error";
5244             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5245                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5246             }
5247         };
5248         if (my $err = $@) {
5249             $add_apply_error->($opt, $err);
5250         } else {
5251             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5252             delete $conf->{$opt};
5253         }
5254     }
5255
5256     PVE::QemuConfig->cleanup_pending($conf);
5257
5258     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5259
5260     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5261         next if $opt eq 'delete'; # just to be sure
5262         eval {
5263             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5264                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5265             }
5266         };
5267         if (my $err = $@) {
5268             $add_apply_error->($opt, $err);
5269         } else {
5270
5271             if (is_valid_drivename($opt)) {
5272                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5273                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5274             }
5275
5276             $conf->{$opt} = delete $conf->{pending}->{$opt};
5277         }
5278     }
5279
5280     # write all changes at once to avoid unnecessary i/o
5281     PVE::QemuConfig->write_config($vmid, $conf);
5282     if ($generate_cloudinit) {
5283         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5284             # After successful generation and if there were changes to be applied, update the
5285             # config to drop the {cloudinit} entry.
5286             PVE::QemuConfig->write_config($vmid, $conf);
5287         }
5288     }
5289 }
5290
5291 sub vmconfig_update_net {
5292     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5293
5294     my $newnet = parse_net($value);
5295
5296     if ($conf->{$opt}) {
5297         my $oldnet = parse_net($conf->{$opt});
5298
5299         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5300             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5301             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5302             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5303
5304             # for non online change, we try to hot-unplug
5305             die "skip\n" if !$hotplug;
5306             vm_deviceunplug($vmid, $conf, $opt);
5307         } else {
5308
5309             die "internal error" if $opt !~ m/net(\d+)/;
5310             my $iface = "tap${vmid}i$1";
5311
5312             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5313                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5314                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5315                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5316                 PVE::Network::tap_unplug($iface);
5317
5318                 if ($have_sdn) {
5319                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5320                 } else {
5321                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5322                 }
5323             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5324                 # Rate can be applied on its own but any change above needs to
5325                 # include the rate in tap_plug since OVS resets everything.
5326                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5327             }
5328
5329             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5330                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5331             }
5332
5333             return 1;
5334         }
5335     }
5336
5337     if ($hotplug) {
5338         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5339     } else {
5340         die "skip\n";
5341     }
5342 }
5343
5344 sub vmconfig_update_agent {
5345     my ($conf, $opt, $value) = @_;
5346
5347     die "skip\n" if !$conf->{$opt};
5348
5349     my $hotplug_options = { fstrim_cloned_disks => 1 };
5350
5351     my $old_agent = parse_guest_agent($conf);
5352     my $agent = parse_guest_agent({$opt => $value});
5353
5354     for my $option (keys %$agent) { # added/changed options
5355         next if defined($hotplug_options->{$option});
5356         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5357     }
5358
5359     for my $option (keys %$old_agent) { # removed options
5360         next if defined($hotplug_options->{$option});
5361         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5362     }
5363
5364     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5365 }
5366
5367 sub vmconfig_update_disk {
5368     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5369
5370     my $drive = parse_drive($opt, $value);
5371
5372     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5373         my $media = $drive->{media} || 'disk';
5374         my $oldmedia = $old_drive->{media} || 'disk';
5375         die "unable to change media type\n" if $media ne $oldmedia;
5376
5377         if (!drive_is_cdrom($old_drive)) {
5378
5379             if ($drive->{file} ne $old_drive->{file}) {
5380
5381                 die "skip\n" if !$hotplug;
5382
5383                 # unplug and register as unused
5384                 vm_deviceunplug($vmid, $conf, $opt);
5385                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5386
5387             } else {
5388                 # update existing disk
5389
5390                 # skip non hotpluggable value
5391                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5392                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5393                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5394                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5395                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5396                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5397                     die "skip\n";
5398                 }
5399
5400                 # apply throttle
5401                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5402                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5403                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5404                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5405                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5406                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5407                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5408                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5409                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5410                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5411                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5412                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5413                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5414                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5415                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5416                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5417                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5418                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5419
5420                     qemu_block_set_io_throttle(
5421                         $vmid,"drive-$opt",
5422                         ($drive->{mbps} || 0)*1024*1024,
5423                         ($drive->{mbps_rd} || 0)*1024*1024,
5424                         ($drive->{mbps_wr} || 0)*1024*1024,
5425                          $drive->{iops} || 0,
5426                          $drive->{iops_rd} || 0,
5427                          $drive->{iops_wr} || 0,
5428                         ($drive->{mbps_max} || 0)*1024*1024,
5429                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5430                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5431                          $drive->{iops_max} || 0,
5432                          $drive->{iops_rd_max} || 0,
5433                          $drive->{iops_wr_max} || 0,
5434                          $drive->{bps_max_length} || 1,
5435                          $drive->{bps_rd_max_length} || 1,
5436                          $drive->{bps_wr_max_length} || 1,
5437                          $drive->{iops_max_length} || 1,
5438                          $drive->{iops_rd_max_length} || 1,
5439                          $drive->{iops_wr_max_length} || 1,
5440                     );
5441
5442                 }
5443
5444                 return 1;
5445             }
5446
5447         } else { # cdrom
5448
5449             if ($drive->{file} eq 'none') {
5450                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5451                 if (drive_is_cloudinit($old_drive)) {
5452                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5453                 }
5454             } else {
5455                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5456
5457                 # force eject if locked
5458                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5459
5460                 if ($path) {
5461                     mon_cmd($vmid, "blockdev-change-medium",
5462                         id => "$opt", filename => "$path");
5463                 }
5464             }
5465
5466             return 1;
5467         }
5468     }
5469
5470     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5471     # hotplug new disks
5472     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5473     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5474 }
5475
5476 sub vmconfig_update_cloudinit_drive {
5477     my ($storecfg, $conf, $vmid) = @_;
5478
5479     my $cloudinit_ds = undef;
5480     my $cloudinit_drive = undef;
5481
5482     PVE::QemuConfig->foreach_volume($conf, sub {
5483         my ($ds, $drive) = @_;
5484         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5485             $cloudinit_ds = $ds;
5486             $cloudinit_drive = $drive;
5487         }
5488     });
5489
5490     return if !$cloudinit_drive;
5491
5492     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5493         PVE::QemuConfig->write_config($vmid, $conf);
5494     }
5495
5496     my $running = PVE::QemuServer::check_running($vmid);
5497
5498     if ($running) {
5499         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5500         if ($path) {
5501             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5502             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5503         }
5504     }
5505 }
5506
5507 # called in locked context by incoming migration
5508 sub vm_migrate_get_nbd_disks {
5509     my ($storecfg, $conf, $replicated_volumes) = @_;
5510
5511     my $local_volumes = {};
5512     PVE::QemuConfig->foreach_volume($conf, sub {
5513         my ($ds, $drive) = @_;
5514
5515         return if drive_is_cdrom($drive);
5516         return if $ds eq 'tpmstate0';
5517
5518         my $volid = $drive->{file};
5519
5520         return if !$volid;
5521
5522         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5523
5524         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5525         return if $scfg->{shared};
5526
5527         # replicated disks re-use existing state via bitmap
5528         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5529         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5530     });
5531     return $local_volumes;
5532 }
5533
5534 # called in locked context by incoming migration
5535 sub vm_migrate_alloc_nbd_disks {
5536     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5537
5538     my $nbd = {};
5539     foreach my $opt (sort keys %$source_volumes) {
5540         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5541
5542         if ($use_existing) {
5543             $nbd->{$opt}->{drivestr} = print_drive($drive);
5544             $nbd->{$opt}->{volid} = $volid;
5545             $nbd->{$opt}->{replicated} = 1;
5546             next;
5547         }
5548
5549         # storage mapping + volname = regular migration
5550         # storage mapping + format = remote migration
5551         # order of precedence, filtered by whether storage supports it:
5552         # 1. explicit requested format
5553         # 2. format of current volume
5554         # 3. default format of storage
5555         if (!$storagemap->{identity}) {
5556             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5557             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5558             if (!$format || !grep { $format eq $_ } @$validFormats) {
5559                 if ($volname) {
5560                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5561                     my $fileFormat = qemu_img_format($scfg, $volname);
5562                     $format = $fileFormat
5563                         if grep { $fileFormat eq $_ } @$validFormats;
5564                 }
5565                 $format //= $defFormat;
5566             }
5567         } else {
5568             # can't happen for remote migration, so $volname is always defined
5569             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5570             $format = qemu_img_format($scfg, $volname);
5571         }
5572
5573         my $size = $drive->{size} / 1024;
5574         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5575         my $newdrive = $drive;
5576         $newdrive->{format} = $format;
5577         $newdrive->{file} = $newvolid;
5578         my $drivestr = print_drive($newdrive);
5579         $nbd->{$opt}->{drivestr} = $drivestr;
5580         $nbd->{$opt}->{volid} = $newvolid;
5581     }
5582
5583     return $nbd;
5584 }
5585
5586 # see vm_start_nolock for parameters, additionally:
5587 # migrate_opts:
5588 #   storagemap = parsed storage map for allocating NBD disks
5589 sub vm_start {
5590     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5591
5592     return PVE::QemuConfig->lock_config($vmid, sub {
5593         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5594
5595         die "you can't start a vm if it's a template\n"
5596             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5597
5598         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5599         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5600
5601         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5602
5603         if ($has_backup_lock && $running) {
5604             # a backup is currently running, attempt to start the guest in the
5605             # existing QEMU instance
5606             return vm_resume($vmid);
5607         }
5608
5609         PVE::QemuConfig->check_lock($conf)
5610             if !($params->{skiplock} || $has_suspended_lock);
5611
5612         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5613
5614         die "VM $vmid already running\n" if $running;
5615
5616         if (my $storagemap = $migrate_opts->{storagemap}) {
5617             my $replicated = $migrate_opts->{replicated_volumes};
5618             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5619             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5620
5621             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5622                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5623             }
5624         }
5625
5626         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5627     });
5628 }
5629
5630
5631 # params:
5632 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5633 #   skiplock => 0/1, skip checking for config lock
5634 #   skiptemplate => 0/1, skip checking whether VM is template
5635 #   forcemachine => to force QEMU machine (rollback/migration)
5636 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5637 #   timeout => in seconds
5638 #   paused => start VM in paused state (backup)
5639 #   resume => resume from hibernation
5640 #   pbs-backing => {
5641 #      sata0 => {
5642 #         repository
5643 #         snapshot
5644 #         keyfile
5645 #         archive
5646 #      },
5647 #      virtio2 => ...
5648 #   }
5649 # migrate_opts:
5650 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5651 #   migratedfrom => source node
5652 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5653 #   network => CIDR of migration network
5654 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5655 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5656 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5657 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5658 #       contained in config
5659 sub vm_start_nolock {
5660     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5661
5662     my $statefile = $params->{statefile};
5663     my $resume = $params->{resume};
5664
5665     my $migratedfrom = $migrate_opts->{migratedfrom};
5666     my $migration_type = $migrate_opts->{type};
5667
5668     my $res = {};
5669
5670     # clean up leftover reboot request files
5671     eval { clear_reboot_request($vmid); };
5672     warn $@ if $@;
5673
5674     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5675         vmconfig_apply_pending($vmid, $conf, $storecfg);
5676         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5677     }
5678
5679     # don't regenerate the ISO if the VM is started as part of a live migration
5680     # this way we can reuse the old ISO with the correct config
5681     if (!$migratedfrom) {
5682         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5683             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5684             # $conf->{cloudinit}, so we could just not do this?
5685             # But we do it above, so for now let's be consistent.
5686             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5687         }
5688     }
5689
5690     # override offline migrated volumes, conf is out of date still
5691     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5692         for my $key (sort keys $offline_volumes->%*) {
5693             my $parsed = parse_drive($key, $conf->{$key});
5694             $parsed->{file} = $offline_volumes->{$key};
5695             $conf->{$key} = print_drive($parsed);
5696         }
5697     }
5698
5699     my $defaults = load_defaults();
5700
5701     # set environment variable useful inside network script
5702     # for remote migration the config is available on the target node!
5703     if (!$migrate_opts->{remote_node}) {
5704         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5705     }
5706
5707     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5708
5709     my $forcemachine = $params->{forcemachine};
5710     my $forcecpu = $params->{forcecpu};
5711     if ($resume) {
5712         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5713         $forcemachine = $conf->{runningmachine};
5714         $forcecpu = $conf->{runningcpu};
5715         print "Resuming suspended VM\n";
5716     }
5717
5718     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5719         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5720
5721     my $migration_ip;
5722     my $get_migration_ip = sub {
5723         my ($nodename) = @_;
5724
5725         return $migration_ip if defined($migration_ip);
5726
5727         my $cidr = $migrate_opts->{network};
5728
5729         if (!defined($cidr)) {
5730             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5731             $cidr = $dc_conf->{migration}->{network};
5732         }
5733
5734         if (defined($cidr)) {
5735             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5736
5737             die "could not get IP: no address configured on local " .
5738                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5739
5740             die "could not get IP: multiple addresses configured on local " .
5741                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5742
5743             $migration_ip = @$ips[0];
5744         }
5745
5746         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5747             if !defined($migration_ip);
5748
5749         return $migration_ip;
5750     };
5751
5752     if ($statefile) {
5753         if ($statefile eq 'tcp') {
5754             my $migrate = $res->{migrate} = { proto => 'tcp' };
5755             $migrate->{addr} = "localhost";
5756             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5757             my $nodename = nodename();
5758
5759             if (!defined($migration_type)) {
5760                 if (defined($datacenterconf->{migration}->{type})) {
5761                     $migration_type = $datacenterconf->{migration}->{type};
5762                 } else {
5763                     $migration_type = 'secure';
5764                 }
5765             }
5766
5767             if ($migration_type eq 'insecure') {
5768                 $migrate->{addr} = $get_migration_ip->($nodename);
5769                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5770             }
5771
5772             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5773             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5774             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5775             push @$cmd, '-incoming', $migrate->{uri};
5776             push @$cmd, '-S';
5777
5778         } elsif ($statefile eq 'unix') {
5779             # should be default for secure migrations as a ssh TCP forward
5780             # tunnel is not deterministic reliable ready and fails regurarly
5781             # to set up in time, so use UNIX socket forwards
5782             my $migrate = $res->{migrate} = { proto => 'unix' };
5783             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5784             unlink $migrate->{addr};
5785
5786             $migrate->{uri} = "unix:$migrate->{addr}";
5787             push @$cmd, '-incoming', $migrate->{uri};
5788             push @$cmd, '-S';
5789
5790         } elsif (-e $statefile) {
5791             push @$cmd, '-loadstate', $statefile;
5792         } else {
5793             my $statepath = PVE::Storage::path($storecfg, $statefile);
5794             push @$vollist, $statefile;
5795             push @$cmd, '-loadstate', $statepath;
5796         }
5797     } elsif ($params->{paused}) {
5798         push @$cmd, '-S';
5799     }
5800
5801     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5802
5803     my $pci_devices = {}; # host pci devices
5804     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5805         my $dev = $conf->{"hostpci$i"} or next;
5806         $pci_devices->{$i} = parse_hostpci($dev);
5807     }
5808
5809     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5810     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5811
5812     # map to a flat list of pci ids
5813     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5814
5815     # reserve all PCI IDs before actually doing anything with them
5816     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5817
5818     eval {
5819         my $uuid;
5820         for my $id (sort keys %$pci_devices) {
5821             my $d = $pci_devices->{$id};
5822             for my $dev ($d->{pciid}->@*) {
5823                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5824
5825                 # nvidia grid needs the uuid of the mdev as qemu parameter
5826                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5827                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5828                 }
5829             }
5830         }
5831         push @$cmd, '-uuid', $uuid if defined($uuid);
5832     };
5833     if (my $err = $@) {
5834         eval { cleanup_pci_devices($vmid, $conf) };
5835         warn $@ if $@;
5836         die $err;
5837     }
5838
5839     PVE::Storage::activate_volumes($storecfg, $vollist);
5840
5841     eval {
5842         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5843     };
5844     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5845     # timeout should be more than enough here...
5846     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5847
5848     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5849
5850     my %run_params = (
5851         timeout => $statefile ? undef : $start_timeout,
5852         umask => 0077,
5853         noerr => 1,
5854     );
5855
5856     # when migrating, prefix QEMU output so other side can pick up any
5857     # errors that might occur and show the user
5858     if ($migratedfrom) {
5859         $run_params{quiet} = 1;
5860         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5861     }
5862
5863     my %systemd_properties = (
5864         Slice => 'qemu.slice',
5865         KillMode => 'process',
5866         SendSIGKILL => 0,
5867         TimeoutStopUSec => ULONG_MAX, # infinity
5868     );
5869
5870     if (PVE::CGroup::cgroup_mode() == 2) {
5871         $systemd_properties{CPUWeight} = $cpuunits;
5872     } else {
5873         $systemd_properties{CPUShares} = $cpuunits;
5874     }
5875
5876     if (my $cpulimit = $conf->{cpulimit}) {
5877         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5878     }
5879     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5880
5881     my $run_qemu = sub {
5882         PVE::Tools::run_fork sub {
5883             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5884
5885             my $tpmpid;
5886             if (my $tpm = $conf->{tpmstate0}) {
5887                 # start the TPM emulator so QEMU can connect on start
5888                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5889             }
5890
5891             my $exitcode = run_command($cmd, %run_params);
5892             if ($exitcode) {
5893                 if ($tpmpid) {
5894                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5895                     kill 'TERM', $tpmpid;
5896                 }
5897                 die "QEMU exited with code $exitcode\n";
5898             }
5899         };
5900     };
5901
5902     if ($conf->{hugepages}) {
5903
5904         my $code = sub {
5905             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5906             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5907
5908             PVE::QemuServer::Memory::hugepages_mount();
5909             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5910
5911             eval { $run_qemu->() };
5912             if (my $err = $@) {
5913                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5914                     if !$conf->{keephugepages};
5915                 die $err;
5916             }
5917
5918             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5919                 if !$conf->{keephugepages};
5920         };
5921         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5922
5923     } else {
5924         eval { $run_qemu->() };
5925     }
5926
5927     if (my $err = $@) {
5928         # deactivate volumes if start fails
5929         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5930         warn $@ if $@;
5931         eval { cleanup_pci_devices($vmid, $conf) };
5932         warn $@ if $@;
5933
5934         die "start failed: $err";
5935     }
5936
5937     # re-reserve all PCI IDs now that we can know the actual VM PID
5938     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5939     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5940     warn $@ if $@;
5941
5942     if (defined($res->{migrate})) {
5943         print "migration listens on $res->{migrate}->{uri}\n";
5944     } elsif ($statefile) {
5945         eval { mon_cmd($vmid, "cont"); };
5946         warn $@ if $@;
5947     }
5948
5949     #start nbd server for storage migration
5950     if (my $nbd = $migrate_opts->{nbd}) {
5951         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5952
5953         my $migrate_storage_uri;
5954         # nbd_protocol_version > 0 for unix socket support
5955         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5956             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5957             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5958             $migrate_storage_uri = "nbd:unix:$socket_path";
5959             $res->{migrate}->{unix_sockets} = [$socket_path];
5960         } else {
5961             my $nodename = nodename();
5962             my $localip = $get_migration_ip->($nodename);
5963             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5964             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5965
5966             mon_cmd($vmid, "nbd-server-start", addr => {
5967                 type => 'inet',
5968                 data => {
5969                     host => "${localip}",
5970                     port => "${storage_migrate_port}",
5971                 },
5972             });
5973             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5974             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5975         }
5976
5977         foreach my $opt (sort keys %$nbd) {
5978             my $drivestr = $nbd->{$opt}->{drivestr};
5979             my $volid = $nbd->{$opt}->{volid};
5980             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5981             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5982             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5983             print "re-using replicated volume: $opt - $volid\n"
5984                 if $nbd->{$opt}->{replicated};
5985
5986             $res->{drives}->{$opt} = $nbd->{$opt};
5987             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5988         }
5989     }
5990
5991     if ($migratedfrom) {
5992         eval {
5993             set_migration_caps($vmid);
5994         };
5995         warn $@ if $@;
5996
5997         if ($spice_port) {
5998             print "spice listens on port $spice_port\n";
5999             $res->{spice_port} = $spice_port;
6000             if ($migrate_opts->{spice_ticket}) {
6001                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6002                     $migrate_opts->{spice_ticket});
6003                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6004             }
6005         }
6006
6007     } else {
6008         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6009             if !$statefile && $conf->{balloon};
6010
6011         foreach my $opt (keys %$conf) {
6012             next if $opt !~  m/^net\d+$/;
6013             my $nicconf = parse_net($conf->{$opt});
6014             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6015         }
6016         add_nets_bridge_fdb($conf, $vmid);
6017     }
6018
6019     mon_cmd($vmid, 'qom-set',
6020                 path => "machine/peripheral/balloon0",
6021                 property => "guest-stats-polling-interval",
6022                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
6023
6024     if ($resume) {
6025         print "Resumed VM, removing state\n";
6026         if (my $vmstate = $conf->{vmstate}) {
6027             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6028             PVE::Storage::vdisk_free($storecfg, $vmstate);
6029         }
6030         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6031         PVE::QemuConfig->write_config($vmid, $conf);
6032     }
6033
6034     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6035
6036     return $res;
6037 }
6038
6039 sub vm_commandline {
6040     my ($storecfg, $vmid, $snapname) = @_;
6041
6042     my $conf = PVE::QemuConfig->load_config($vmid);
6043
6044     my ($forcemachine, $forcecpu);
6045     if ($snapname) {
6046         my $snapshot = $conf->{snapshots}->{$snapname};
6047         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6048
6049         # check for machine or CPU overrides in snapshot
6050         $forcemachine = $snapshot->{runningmachine};
6051         $forcecpu = $snapshot->{runningcpu};
6052
6053         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6054
6055         $conf = $snapshot;
6056     }
6057
6058     my $defaults = load_defaults();
6059
6060     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6061
6062     return PVE::Tools::cmd2string($cmd);
6063 }
6064
6065 sub vm_reset {
6066     my ($vmid, $skiplock) = @_;
6067
6068     PVE::QemuConfig->lock_config($vmid, sub {
6069
6070         my $conf = PVE::QemuConfig->load_config($vmid);
6071
6072         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6073
6074         mon_cmd($vmid, "system_reset");
6075     });
6076 }
6077
6078 sub get_vm_volumes {
6079     my ($conf) = @_;
6080
6081     my $vollist = [];
6082     foreach_volid($conf, sub {
6083         my ($volid, $attr) = @_;
6084
6085         return if $volid =~ m|^/|;
6086
6087         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6088         return if !$sid;
6089
6090         push @$vollist, $volid;
6091     });
6092
6093     return $vollist;
6094 }
6095
6096 sub cleanup_pci_devices {
6097     my ($vmid, $conf) = @_;
6098
6099     foreach my $key (keys %$conf) {
6100         next if $key !~ m/^hostpci(\d+)$/;
6101         my $hostpciindex = $1;
6102         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6103         my $d = parse_hostpci($conf->{$key});
6104         if ($d->{mdev}) {
6105             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6106             # don't want to break ABI just for this two liner
6107             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6108             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6109         }
6110     }
6111     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6112 }
6113
6114 sub vm_stop_cleanup {
6115     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6116
6117     eval {
6118
6119         if (!$keepActive) {
6120             my $vollist = get_vm_volumes($conf);
6121             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6122
6123             if (my $tpmdrive = $conf->{tpmstate0}) {
6124                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6125                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6126                 if ($storeid) {
6127                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6128                 }
6129             }
6130         }
6131
6132         foreach my $ext (qw(mon qmp pid vnc qga)) {
6133             unlink "/var/run/qemu-server/${vmid}.$ext";
6134         }
6135
6136         if ($conf->{ivshmem}) {
6137             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6138             # just delete it for now, VMs which have this already open do not
6139             # are affected, but new VMs will get a separated one. If this
6140             # becomes an issue we either add some sort of ref-counting or just
6141             # add a "don't delete on stop" flag to the ivshmem format.
6142             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6143         }
6144
6145         cleanup_pci_devices($vmid, $conf);
6146
6147         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6148     };
6149     warn $@ if $@; # avoid errors - just warn
6150 }
6151
6152 # call only in locked context
6153 sub _do_vm_stop {
6154     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6155
6156     my $pid = check_running($vmid, $nocheck);
6157     return if !$pid;
6158
6159     my $conf;
6160     if (!$nocheck) {
6161         $conf = PVE::QemuConfig->load_config($vmid);
6162         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6163         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6164             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6165             $timeout = $opts->{down} if $opts->{down};
6166         }
6167         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6168     }
6169
6170     eval {
6171         if ($shutdown) {
6172             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6173                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6174             } else {
6175                 mon_cmd($vmid, "system_powerdown");
6176             }
6177         } else {
6178             mon_cmd($vmid, "quit");
6179         }
6180     };
6181     my $err = $@;
6182
6183     if (!$err) {
6184         $timeout = 60 if !defined($timeout);
6185
6186         my $count = 0;
6187         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6188             $count++;
6189             sleep 1;
6190         }
6191
6192         if ($count >= $timeout) {
6193             if ($force) {
6194                 warn "VM still running - terminating now with SIGTERM\n";
6195                 kill 15, $pid;
6196             } else {
6197                 die "VM quit/powerdown failed - got timeout\n";
6198             }
6199         } else {
6200             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6201             return;
6202         }
6203     } else {
6204        if (!check_running($vmid, $nocheck)) {
6205             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6206             return;
6207        }
6208        if ($force) {
6209             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6210             kill 15, $pid;
6211         } else {
6212             die "VM quit/powerdown failed\n";
6213         }
6214     }
6215
6216     # wait again
6217     $timeout = 10;
6218
6219     my $count = 0;
6220     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6221         $count++;
6222         sleep 1;
6223     }
6224
6225     if ($count >= $timeout) {
6226         warn "VM still running - terminating now with SIGKILL\n";
6227         kill 9, $pid;
6228         sleep 1;
6229     }
6230
6231     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6232 }
6233
6234 # Note: use $nocheck to skip tests if VM configuration file exists.
6235 # We need that when migration VMs to other nodes (files already moved)
6236 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6237 sub vm_stop {
6238     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6239
6240     $force = 1 if !defined($force) && !$shutdown;
6241
6242     if ($migratedfrom){
6243         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6244         kill 15, $pid if $pid;
6245         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6246         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6247         return;
6248     }
6249
6250     PVE::QemuConfig->lock_config($vmid, sub {
6251         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6252    });
6253 }
6254
6255 sub vm_reboot {
6256     my ($vmid, $timeout) = @_;
6257
6258     PVE::QemuConfig->lock_config($vmid, sub {
6259         eval {
6260
6261             # only reboot if running, as qmeventd starts it again on a stop event
6262             return if !check_running($vmid);
6263
6264             create_reboot_request($vmid);
6265
6266             my $storecfg = PVE::Storage::config();
6267             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6268
6269         };
6270         if (my $err = $@) {
6271             # avoid that the next normal shutdown will be confused for a reboot
6272             clear_reboot_request($vmid);
6273             die $err;
6274         }
6275    });
6276 }
6277
6278 # note: if using the statestorage parameter, the caller has to check privileges
6279 sub vm_suspend {
6280     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6281
6282     my $conf;
6283     my $path;
6284     my $storecfg;
6285     my $vmstate;
6286
6287     PVE::QemuConfig->lock_config($vmid, sub {
6288
6289         $conf = PVE::QemuConfig->load_config($vmid);
6290
6291         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6292         PVE::QemuConfig->check_lock($conf)
6293             if !($skiplock || $is_backing_up);
6294
6295         die "cannot suspend to disk during backup\n"
6296             if $is_backing_up && $includestate;
6297
6298         if ($includestate) {
6299             $conf->{lock} = 'suspending';
6300             my $date = strftime("%Y-%m-%d", localtime(time()));
6301             $storecfg = PVE::Storage::config();
6302             if (!$statestorage) {
6303                 $statestorage = find_vmstate_storage($conf, $storecfg);
6304                 # check permissions for the storage
6305                 my $rpcenv = PVE::RPCEnvironment::get();
6306                 if ($rpcenv->{type} ne 'cli') {
6307                     my $authuser = $rpcenv->get_user();
6308                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6309                 }
6310             }
6311
6312
6313             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6314                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6315             $path = PVE::Storage::path($storecfg, $vmstate);
6316             PVE::QemuConfig->write_config($vmid, $conf);
6317         } else {
6318             mon_cmd($vmid, "stop");
6319         }
6320     });
6321
6322     if ($includestate) {
6323         # save vm state
6324         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6325
6326         eval {
6327             set_migration_caps($vmid, 1);
6328             mon_cmd($vmid, "savevm-start", statefile => $path);
6329             for(;;) {
6330                 my $state = mon_cmd($vmid, "query-savevm");
6331                 if (!$state->{status}) {
6332                     die "savevm not active\n";
6333                 } elsif ($state->{status} eq 'active') {
6334                     sleep(1);
6335                     next;
6336                 } elsif ($state->{status} eq 'completed') {
6337                     print "State saved, quitting\n";
6338                     last;
6339                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6340                     die "query-savevm failed with error '$state->{error}'\n"
6341                 } else {
6342                     die "query-savevm returned status '$state->{status}'\n";
6343                 }
6344             }
6345         };
6346         my $err = $@;
6347
6348         PVE::QemuConfig->lock_config($vmid, sub {
6349             $conf = PVE::QemuConfig->load_config($vmid);
6350             if ($err) {
6351                 # cleanup, but leave suspending lock, to indicate something went wrong
6352                 eval {
6353                     mon_cmd($vmid, "savevm-end");
6354                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6355                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6356                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6357                     PVE::QemuConfig->write_config($vmid, $conf);
6358                 };
6359                 warn $@ if $@;
6360                 die $err;
6361             }
6362
6363             die "lock changed unexpectedly\n"
6364                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6365
6366             mon_cmd($vmid, "quit");
6367             $conf->{lock} = 'suspended';
6368             PVE::QemuConfig->write_config($vmid, $conf);
6369         });
6370     }
6371 }
6372
6373 # $nocheck is set when called as part of a migration - in this context the
6374 # location of the config file (source or target node) is not deterministic,
6375 # since migration cannot wait for pmxcfs to process the rename
6376 sub vm_resume {
6377     my ($vmid, $skiplock, $nocheck) = @_;
6378
6379     PVE::QemuConfig->lock_config($vmid, sub {
6380         my $res = mon_cmd($vmid, 'query-status');
6381         my $resume_cmd = 'cont';
6382         my $reset = 0;
6383         my $conf;
6384         if ($nocheck) {
6385             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6386             if ($@) {
6387                 my $vmlist = PVE::Cluster::get_vmlist();
6388                 if (exists($vmlist->{ids}->{$vmid})) {
6389                     my $node = $vmlist->{ids}->{$vmid}->{node};
6390                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6391                 }
6392                 if (!$conf) {
6393                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6394                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6395                 }
6396             }
6397         } else {
6398             $conf = PVE::QemuConfig->load_config($vmid);
6399         }
6400
6401         if ($res->{status}) {
6402             return if $res->{status} eq 'running'; # job done, go home
6403             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6404             $reset = 1 if $res->{status} eq 'shutdown';
6405         }
6406
6407         if (!$nocheck) {
6408             PVE::QemuConfig->check_lock($conf)
6409                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6410         }
6411
6412         if ($reset) {
6413             # required if a VM shuts down during a backup and we get a resume
6414             # request before the backup finishes for example
6415             mon_cmd($vmid, "system_reset");
6416         }
6417
6418         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6419
6420         mon_cmd($vmid, $resume_cmd);
6421     });
6422 }
6423
6424 sub vm_sendkey {
6425     my ($vmid, $skiplock, $key) = @_;
6426
6427     PVE::QemuConfig->lock_config($vmid, sub {
6428
6429         my $conf = PVE::QemuConfig->load_config($vmid);
6430
6431         # there is no qmp command, so we use the human monitor command
6432         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6433         die $res if $res ne '';
6434     });
6435 }
6436
6437 # vzdump restore implementaion
6438
6439 sub tar_archive_read_firstfile {
6440     my $archive = shift;
6441
6442     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6443
6444     # try to detect archive type first
6445     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6446         die "unable to open file '$archive'\n";
6447     my $firstfile = <$fh>;
6448     kill 15, $pid;
6449     close $fh;
6450
6451     die "ERROR: archive contaions no data\n" if !$firstfile;
6452     chomp $firstfile;
6453
6454     return $firstfile;
6455 }
6456
6457 sub tar_restore_cleanup {
6458     my ($storecfg, $statfile) = @_;
6459
6460     print STDERR "starting cleanup\n";
6461
6462     if (my $fd = IO::File->new($statfile, "r")) {
6463         while (defined(my $line = <$fd>)) {
6464             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6465                 my $volid = $2;
6466                 eval {
6467                     if ($volid =~ m|^/|) {
6468                         unlink $volid || die 'unlink failed\n';
6469                     } else {
6470                         PVE::Storage::vdisk_free($storecfg, $volid);
6471                     }
6472                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6473                 };
6474                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6475             } else {
6476                 print STDERR "unable to parse line in statfile - $line";
6477             }
6478         }
6479         $fd->close();
6480     }
6481 }
6482
6483 sub restore_file_archive {
6484     my ($archive, $vmid, $user, $opts) = @_;
6485
6486     return restore_vma_archive($archive, $vmid, $user, $opts)
6487         if $archive eq '-';
6488
6489     my $info = PVE::Storage::archive_info($archive);
6490     my $format = $opts->{format} // $info->{format};
6491     my $comp = $info->{compression};
6492
6493     # try to detect archive format
6494     if ($format eq 'tar') {
6495         return restore_tar_archive($archive, $vmid, $user, $opts);
6496     } else {
6497         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6498     }
6499 }
6500
6501 # hepler to remove disks that will not be used after restore
6502 my $restore_cleanup_oldconf = sub {
6503     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6504
6505     my $kept_disks = {};
6506
6507     PVE::QemuConfig->foreach_volume($oldconf, sub {
6508         my ($ds, $drive) = @_;
6509
6510         return if drive_is_cdrom($drive, 1);
6511
6512         my $volid = $drive->{file};
6513         return if !$volid || $volid =~ m|^/|;
6514
6515         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6516         return if !$path || !$owner || ($owner != $vmid);
6517
6518         # Note: only delete disk we want to restore
6519         # other volumes will become unused
6520         if ($virtdev_hash->{$ds}) {
6521             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6522             if (my $err = $@) {
6523                 warn $err;
6524             }
6525         } else {
6526             $kept_disks->{$volid} = 1;
6527         }
6528     });
6529
6530     # after the restore we have no snapshots anymore
6531     for my $snapname (keys $oldconf->{snapshots}->%*) {
6532         my $snap = $oldconf->{snapshots}->{$snapname};
6533         if ($snap->{vmstate}) {
6534             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6535             if (my $err = $@) {
6536                 warn $err;
6537             }
6538         }
6539
6540         for my $volid (keys $kept_disks->%*) {
6541             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6542             warn $@ if $@;
6543         }
6544     }
6545 };
6546
6547 # Helper to parse vzdump backup device hints
6548 #
6549 # $rpcenv: Environment, used to ckeck storage permissions
6550 # $user: User ID, to check storage permissions
6551 # $storecfg: Storage configuration
6552 # $fh: the file handle for reading the configuration
6553 # $devinfo: should contain device sizes for all backu-up'ed devices
6554 # $options: backup options (pool, default storage)
6555 #
6556 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6557 my $parse_backup_hints = sub {
6558     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6559
6560     my $check_storage = sub { # assert if an image can be allocate
6561         my ($storeid, $scfg) = @_;
6562         die "Content type 'images' is not available on storage '$storeid'\n"
6563             if !$scfg->{content}->{images};
6564         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6565             if $user ne 'root@pam';
6566     };
6567
6568     my $virtdev_hash = {};
6569     while (defined(my $line = <$fh>)) {
6570         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6571             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6572             die "archive does not contain data for drive '$virtdev'\n"
6573                 if !$devinfo->{$devname};
6574
6575             if (defined($options->{storage})) {
6576                 $storeid = $options->{storage} || 'local';
6577             } elsif (!$storeid) {
6578                 $storeid = 'local';
6579             }
6580             $format = 'raw' if !$format;
6581             $devinfo->{$devname}->{devname} = $devname;
6582             $devinfo->{$devname}->{virtdev} = $virtdev;
6583             $devinfo->{$devname}->{format} = $format;
6584             $devinfo->{$devname}->{storeid} = $storeid;
6585
6586             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6587             $check_storage->($storeid, $scfg); # permission and content type check
6588
6589             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6590         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6591             my $virtdev = $1;
6592             my $drive = parse_drive($virtdev, $2);
6593
6594             if (drive_is_cloudinit($drive)) {
6595                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6596                 $storeid = $options->{storage} if defined ($options->{storage});
6597                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6598                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6599
6600                 $check_storage->($storeid, $scfg); # permission and content type check
6601
6602                 $virtdev_hash->{$virtdev} = {
6603                     format => $format,
6604                     storeid => $storeid,
6605                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6606                     is_cloudinit => 1,
6607                 };
6608             }
6609         }
6610     }
6611
6612     return $virtdev_hash;
6613 };
6614
6615 # Helper to allocate and activate all volumes required for a restore
6616 #
6617 # $storecfg: Storage configuration
6618 # $virtdev_hash: as returned by parse_backup_hints()
6619 #
6620 # Returns: { $virtdev => $volid }
6621 my $restore_allocate_devices = sub {
6622     my ($storecfg, $virtdev_hash, $vmid) = @_;
6623
6624     my $map = {};
6625     foreach my $virtdev (sort keys %$virtdev_hash) {
6626         my $d = $virtdev_hash->{$virtdev};
6627         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6628         my $storeid = $d->{storeid};
6629         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6630
6631         # test if requested format is supported
6632         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6633         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6634         $d->{format} = $defFormat if !$supported;
6635
6636         my $name;
6637         if ($d->{is_cloudinit}) {
6638             $name = "vm-$vmid-cloudinit";
6639             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6640             if ($scfg->{path}) {
6641                 $name .= ".$d->{format}";
6642             }
6643         }
6644
6645         my $volid = PVE::Storage::vdisk_alloc(
6646             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6647
6648         print STDERR "new volume ID is '$volid'\n";
6649         $d->{volid} = $volid;
6650
6651         PVE::Storage::activate_volumes($storecfg, [$volid]);
6652
6653         $map->{$virtdev} = $volid;
6654     }
6655
6656     return $map;
6657 };
6658
6659 sub restore_update_config_line {
6660     my ($cookie, $map, $line, $unique) = @_;
6661
6662     return '' if $line =~ m/^\#qmdump\#/;
6663     return '' if $line =~ m/^\#vzdump\#/;
6664     return '' if $line =~ m/^lock:/;
6665     return '' if $line =~ m/^unused\d+:/;
6666     return '' if $line =~ m/^parent:/;
6667
6668     my $res = '';
6669
6670     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6671     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6672         # try to convert old 1.X settings
6673         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6674         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6675             my ($model, $macaddr) = split(/\=/, $devconfig);
6676             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6677             my $net = {
6678                 model => $model,
6679                 bridge => "vmbr$ind",
6680                 macaddr => $macaddr,
6681             };
6682             my $netstr = print_net($net);
6683
6684             $res .= "net$cookie->{netcount}: $netstr\n";
6685             $cookie->{netcount}++;
6686         }
6687     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6688         my ($id, $netstr) = ($1, $2);
6689         my $net = parse_net($netstr);
6690         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6691         $netstr = print_net($net);
6692         $res .= "$id: $netstr\n";
6693     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6694         my $virtdev = $1;
6695         my $value = $3;
6696         my $di = parse_drive($virtdev, $value);
6697         if (defined($di->{backup}) && !$di->{backup}) {
6698             $res .= "#$line";
6699         } elsif ($map->{$virtdev}) {
6700             delete $di->{format}; # format can change on restore
6701             $di->{file} = $map->{$virtdev};
6702             $value = print_drive($di);
6703             $res .= "$virtdev: $value\n";
6704         } else {
6705             $res .= $line;
6706         }
6707     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6708         my $vmgenid = $1;
6709         if ($vmgenid ne '0') {
6710             # always generate a new vmgenid if there was a valid one setup
6711             $vmgenid = generate_uuid();
6712         }
6713         $res .= "vmgenid: $vmgenid\n";
6714     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6715         my ($uuid, $uuid_str);
6716         UUID::generate($uuid);
6717         UUID::unparse($uuid, $uuid_str);
6718         my $smbios1 = parse_smbios1($2);
6719         $smbios1->{uuid} = $uuid_str;
6720         $res .= $1.print_smbios1($smbios1)."\n";
6721     } else {
6722         $res .= $line;
6723     }
6724
6725     return $res;
6726 }
6727
6728 my $restore_deactivate_volumes = sub {
6729     my ($storecfg, $virtdev_hash) = @_;
6730
6731     my $vollist = [];
6732     for my $dev (values $virtdev_hash->%*) {
6733         push $vollist->@*, $dev->{volid} if $dev->{volid};
6734     }
6735
6736     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6737     print STDERR $@ if $@;
6738 };
6739
6740 my $restore_destroy_volumes = sub {
6741     my ($storecfg, $virtdev_hash) = @_;
6742
6743     for my $dev (values $virtdev_hash->%*) {
6744         my $volid = $dev->{volid} or next;
6745         eval {
6746             PVE::Storage::vdisk_free($storecfg, $volid);
6747             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6748         };
6749         print STDERR "unable to cleanup '$volid' - $@" if $@;
6750     }
6751 };
6752
6753 my $restore_merge_config = sub {
6754     my ($filename, $backup_conf_raw, $override_conf) = @_;
6755
6756     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6757     for my $key (keys $override_conf->%*) {
6758         $backup_conf->{$key} = $override_conf->{$key};
6759     }
6760
6761     return $backup_conf;
6762 };
6763
6764 sub scan_volids {
6765     my ($cfg, $vmid) = @_;
6766
6767     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6768
6769     my $volid_hash = {};
6770     foreach my $storeid (keys %$info) {
6771         foreach my $item (@{$info->{$storeid}}) {
6772             next if !($item->{volid} && $item->{size});
6773             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6774             $volid_hash->{$item->{volid}} = $item;
6775         }
6776     }
6777
6778     return $volid_hash;
6779 }
6780
6781 sub update_disk_config {
6782     my ($vmid, $conf, $volid_hash) = @_;
6783
6784     my $changes;
6785     my $prefix = "VM $vmid";
6786
6787     # used and unused disks
6788     my $referenced = {};
6789
6790     # Note: it is allowed to define multiple storages with same path (alias), so
6791     # we need to check both 'volid' and real 'path' (two different volid can point
6792     # to the same path).
6793
6794     my $referencedpath = {};
6795
6796     # update size info
6797     PVE::QemuConfig->foreach_volume($conf, sub {
6798         my ($opt, $drive) = @_;
6799
6800         my $volid = $drive->{file};
6801         return if !$volid;
6802         my $volume = $volid_hash->{$volid};
6803
6804         # mark volid as "in-use" for next step
6805         $referenced->{$volid} = 1;
6806         if ($volume && (my $path = $volume->{path})) {
6807             $referencedpath->{$path} = 1;
6808         }
6809
6810         return if drive_is_cdrom($drive);
6811         return if !$volume;
6812
6813         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6814         if (defined($updated)) {
6815             $changes = 1;
6816             $conf->{$opt} = print_drive($updated);
6817             print "$prefix ($opt): $msg\n";
6818         }
6819     });
6820
6821     # remove 'unusedX' entry if volume is used
6822     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6823         my ($opt, $drive) = @_;
6824
6825         my $volid = $drive->{file};
6826         return if !$volid;
6827
6828         my $path;
6829         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6830         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6831             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6832             $changes = 1;
6833             delete $conf->{$opt};
6834         }
6835
6836         $referenced->{$volid} = 1;
6837         $referencedpath->{$path} = 1 if $path;
6838     });
6839
6840     foreach my $volid (sort keys %$volid_hash) {
6841         next if $volid =~ m/vm-$vmid-state-/;
6842         next if $referenced->{$volid};
6843         my $path = $volid_hash->{$volid}->{path};
6844         next if !$path; # just to be sure
6845         next if $referencedpath->{$path};
6846         $changes = 1;
6847         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6848         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6849         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6850     }
6851
6852     return $changes;
6853 }
6854
6855 sub rescan {
6856     my ($vmid, $nolock, $dryrun) = @_;
6857
6858     my $cfg = PVE::Storage::config();
6859
6860     print "rescan volumes...\n";
6861     my $volid_hash = scan_volids($cfg, $vmid);
6862
6863     my $updatefn =  sub {
6864         my ($vmid) = @_;
6865
6866         my $conf = PVE::QemuConfig->load_config($vmid);
6867
6868         PVE::QemuConfig->check_lock($conf);
6869
6870         my $vm_volids = {};
6871         foreach my $volid (keys %$volid_hash) {
6872             my $info = $volid_hash->{$volid};
6873             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6874         }
6875
6876         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6877
6878         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6879     };
6880
6881     if (defined($vmid)) {
6882         if ($nolock) {
6883             &$updatefn($vmid);
6884         } else {
6885             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6886         }
6887     } else {
6888         my $vmlist = config_list();
6889         foreach my $vmid (keys %$vmlist) {
6890             if ($nolock) {
6891                 &$updatefn($vmid);
6892             } else {
6893                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6894             }
6895         }
6896     }
6897 }
6898
6899 sub restore_proxmox_backup_archive {
6900     my ($archive, $vmid, $user, $options) = @_;
6901
6902     my $storecfg = PVE::Storage::config();
6903
6904     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6905     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6906
6907     my $fingerprint = $scfg->{fingerprint};
6908     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6909
6910     my $repo = PVE::PBSClient::get_repository($scfg);
6911     my $namespace = $scfg->{namespace};
6912
6913     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6914     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6915     local $ENV{PBS_PASSWORD} = $password;
6916     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6917
6918     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6919         PVE::Storage::parse_volname($storecfg, $archive);
6920
6921     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6922
6923     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6924
6925     my $tmpdir = "/var/tmp/vzdumptmp$$";
6926     rmtree $tmpdir;
6927     mkpath $tmpdir;
6928
6929     my $conffile = PVE::QemuConfig->config_file($vmid);
6930      # disable interrupts (always do cleanups)
6931     local $SIG{INT} =
6932         local $SIG{TERM} =
6933         local $SIG{QUIT} =
6934         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6935
6936     # Note: $oldconf is undef if VM does not exists
6937     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6938     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6939     my $new_conf_raw = '';
6940
6941     my $rpcenv = PVE::RPCEnvironment::get();
6942     my $devinfo = {}; # info about drives included in backup
6943     my $virtdev_hash = {}; # info about allocated drives
6944
6945     eval {
6946         # enable interrupts
6947         local $SIG{INT} =
6948             local $SIG{TERM} =
6949             local $SIG{QUIT} =
6950             local $SIG{HUP} =
6951             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6952
6953         my $cfgfn = "$tmpdir/qemu-server.conf";
6954         my $firewall_config_fn = "$tmpdir/fw.conf";
6955         my $index_fn = "$tmpdir/index.json";
6956
6957         my $cmd = "restore";
6958
6959         my $param = [$pbs_backup_name, "index.json", $index_fn];
6960         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6961         my $index = PVE::Tools::file_get_contents($index_fn);
6962         $index = decode_json($index);
6963
6964         foreach my $info (@{$index->{files}}) {
6965             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6966                 my $devname = $1;
6967                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6968                     $devinfo->{$devname}->{size} = $1;
6969                 } else {
6970                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6971                 }
6972             }
6973         }
6974
6975         my $is_qemu_server_backup = scalar(
6976             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6977         );
6978         if (!$is_qemu_server_backup) {
6979             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6980         }
6981         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6982
6983         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6984         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6985
6986         if ($has_firewall_config) {
6987             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6988             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6989
6990             my $pve_firewall_dir = '/etc/pve/firewall';
6991             mkdir $pve_firewall_dir; # make sure the dir exists
6992             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6993         }
6994
6995         my $fh = IO::File->new($cfgfn, "r") ||
6996             die "unable to read qemu-server.conf - $!\n";
6997
6998         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6999
7000         # fixme: rate limit?
7001
7002         # create empty/temp config
7003         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7004
7005         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7006
7007         # allocate volumes
7008         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7009
7010         foreach my $virtdev (sort keys %$virtdev_hash) {
7011             my $d = $virtdev_hash->{$virtdev};
7012             next if $d->{is_cloudinit}; # no need to restore cloudinit
7013
7014             # this fails if storage is unavailable
7015             my $volid = $d->{volid};
7016             my $path = PVE::Storage::path($storecfg, $volid);
7017
7018             # for live-restore we only want to preload the efidisk and TPM state
7019             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7020
7021             my @ns_arg;
7022             if (defined(my $ns = $scfg->{namespace})) {
7023                 @ns_arg = ('--ns', $ns);
7024             }
7025
7026             my $pbs_restore_cmd = [
7027                 '/usr/bin/pbs-restore',
7028                 '--repository', $repo,
7029                 @ns_arg,
7030                 $pbs_backup_name,
7031                 "$d->{devname}.img.fidx",
7032                 $path,
7033                 '--verbose',
7034                 ];
7035
7036             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7037             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7038
7039             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7040                 push @$pbs_restore_cmd, '--skip-zero';
7041             }
7042
7043             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7044             print "restore proxmox backup image: $dbg_cmdstring\n";
7045             run_command($pbs_restore_cmd);
7046         }
7047
7048         $fh->seek(0, 0) || die "seek failed - $!\n";
7049
7050         my $cookie = { netcount => 0 };
7051         while (defined(my $line = <$fh>)) {
7052             $new_conf_raw .= restore_update_config_line(
7053                 $cookie,
7054                 $map,
7055                 $line,
7056                 $options->{unique},
7057             );
7058         }
7059
7060         $fh->close();
7061     };
7062     my $err = $@;
7063
7064     if ($err || !$options->{live}) {
7065         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7066     }
7067
7068     rmtree $tmpdir;
7069
7070     if ($err) {
7071         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7072         die $err;
7073     }
7074
7075     if ($options->{live}) {
7076         # keep lock during live-restore
7077         $new_conf_raw .= "\nlock: create";
7078     }
7079
7080     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7081     PVE::QemuConfig->write_config($vmid, $new_conf);
7082
7083     eval { rescan($vmid, 1); };
7084     warn $@ if $@;
7085
7086     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7087
7088     if ($options->{live}) {
7089         # enable interrupts
7090         local $SIG{INT} =
7091             local $SIG{TERM} =
7092             local $SIG{QUIT} =
7093             local $SIG{HUP} =
7094             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7095
7096         my $conf = PVE::QemuConfig->load_config($vmid);
7097         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7098
7099         # these special drives are already restored before start
7100         delete $devinfo->{'drive-efidisk0'};
7101         delete $devinfo->{'drive-tpmstate0-backup'};
7102
7103         my $pbs_opts = {
7104             repo => $repo,
7105             keyfile => $keyfile,
7106             snapshot => $pbs_backup_name,
7107             namespace => $namespace,
7108         };
7109         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7110
7111         PVE::QemuConfig->remove_lock($vmid, "create");
7112     }
7113 }
7114
7115 sub pbs_live_restore {
7116     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7117
7118     print "starting VM for live-restore\n";
7119     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7120
7121     my $pbs_backing = {};
7122     for my $ds (keys %$restored_disks) {
7123         $ds =~ m/^drive-(.*)$/;
7124         my $confname = $1;
7125         $pbs_backing->{$confname} = {
7126             repository => $opts->{repo},
7127             snapshot => $opts->{snapshot},
7128             archive => "$ds.img.fidx",
7129         };
7130         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7131         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7132
7133         my $drive = parse_drive($confname, $conf->{$confname});
7134         print "restoring '$ds' to '$drive->{file}'\n";
7135     }
7136
7137     my $drives_streamed = 0;
7138     eval {
7139         # make sure HA doesn't interrupt our restore by stopping the VM
7140         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7141             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7142         }
7143
7144         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7145         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7146         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7147
7148         my $qmeventd_fd = register_qmeventd_handle($vmid);
7149
7150         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7151         # this will effectively collapse the backing image chain consisting of
7152         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7153         # removes itself once all backing images vanish with 'auto-remove=on')
7154         my $jobs = {};
7155         for my $ds (sort keys %$restored_disks) {
7156             my $job_id = "restore-$ds";
7157             mon_cmd($vmid, 'block-stream',
7158                 'job-id' => $job_id,
7159                 device => "$ds",
7160             );
7161             $jobs->{$job_id} = {};
7162         }
7163
7164         mon_cmd($vmid, 'cont');
7165         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7166
7167         print "restore-drive jobs finished successfully, removing all tracking block devices"
7168             ." to disconnect from Proxmox Backup Server\n";
7169
7170         for my $ds (sort keys %$restored_disks) {
7171             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7172         }
7173
7174         close($qmeventd_fd);
7175     };
7176
7177     my $err = $@;
7178
7179     if ($err) {
7180         warn "An error occurred during live-restore: $err\n";
7181         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7182         die "live-restore failed\n";
7183     }
7184 }
7185
7186 sub restore_vma_archive {
7187     my ($archive, $vmid, $user, $opts, $comp) = @_;
7188
7189     my $readfrom = $archive;
7190
7191     my $cfg = PVE::Storage::config();
7192     my $commands = [];
7193     my $bwlimit = $opts->{bwlimit};
7194
7195     my $dbg_cmdstring = '';
7196     my $add_pipe = sub {
7197         my ($cmd) = @_;
7198         push @$commands, $cmd;
7199         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7200         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7201         $readfrom = '-';
7202     };
7203
7204     my $input = undef;
7205     if ($archive eq '-') {
7206         $input = '<&STDIN';
7207     } else {
7208         # If we use a backup from a PVE defined storage we also consider that
7209         # storage's rate limit:
7210         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7211         if (defined($volid)) {
7212             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7213             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7214             if ($readlimit) {
7215                 print STDERR "applying read rate limit: $readlimit\n";
7216                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7217                 $add_pipe->($cstream);
7218             }
7219         }
7220     }
7221
7222     if ($comp) {
7223         my $info = PVE::Storage::decompressor_info('vma', $comp);
7224         my $cmd = $info->{decompressor};
7225         push @$cmd, $readfrom;
7226         $add_pipe->($cmd);
7227     }
7228
7229     my $tmpdir = "/var/tmp/vzdumptmp$$";
7230     rmtree $tmpdir;
7231
7232     # disable interrupts (always do cleanups)
7233     local $SIG{INT} =
7234         local $SIG{TERM} =
7235         local $SIG{QUIT} =
7236         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7237
7238     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7239     POSIX::mkfifo($mapfifo, 0600);
7240     my $fifofh;
7241     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7242
7243     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7244
7245     my $oldtimeout;
7246     my $timeout = 5;
7247
7248     my $devinfo = {}; # info about drives included in backup
7249     my $virtdev_hash = {}; # info about allocated drives
7250
7251     my $rpcenv = PVE::RPCEnvironment::get();
7252
7253     my $conffile = PVE::QemuConfig->config_file($vmid);
7254
7255     # Note: $oldconf is undef if VM does not exist
7256     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7257     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7258     my $new_conf_raw = '';
7259
7260     my %storage_limits;
7261
7262     my $print_devmap = sub {
7263         my $cfgfn = "$tmpdir/qemu-server.conf";
7264
7265         # we can read the config - that is already extracted
7266         my $fh = IO::File->new($cfgfn, "r") ||
7267             die "unable to read qemu-server.conf - $!\n";
7268
7269         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7270         if (-f $fwcfgfn) {
7271             my $pve_firewall_dir = '/etc/pve/firewall';
7272             mkdir $pve_firewall_dir; # make sure the dir exists
7273             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7274         }
7275
7276         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7277
7278         foreach my $info (values %{$virtdev_hash}) {
7279             my $storeid = $info->{storeid};
7280             next if defined($storage_limits{$storeid});
7281
7282             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7283             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7284             $storage_limits{$storeid} = $limit * 1024;
7285         }
7286
7287         foreach my $devname (keys %$devinfo) {
7288             die "found no device mapping information for device '$devname'\n"
7289                 if !$devinfo->{$devname}->{virtdev};
7290         }
7291
7292         # create empty/temp config
7293         if ($oldconf) {
7294             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7295             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7296         }
7297
7298         # allocate volumes
7299         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7300
7301         # print restore information to $fifofh
7302         foreach my $virtdev (sort keys %$virtdev_hash) {
7303             my $d = $virtdev_hash->{$virtdev};
7304             next if $d->{is_cloudinit}; # no need to restore cloudinit
7305
7306             my $storeid = $d->{storeid};
7307             my $volid = $d->{volid};
7308
7309             my $map_opts = '';
7310             if (my $limit = $storage_limits{$storeid}) {
7311                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7312             }
7313
7314             my $write_zeros = 1;
7315             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7316                 $write_zeros = 0;
7317             }
7318
7319             my $path = PVE::Storage::path($cfg, $volid);
7320
7321             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7322
7323             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7324         }
7325
7326         $fh->seek(0, 0) || die "seek failed - $!\n";
7327
7328         my $cookie = { netcount => 0 };
7329         while (defined(my $line = <$fh>)) {
7330             $new_conf_raw .= restore_update_config_line(
7331                 $cookie,
7332                 $map,
7333                 $line,
7334                 $opts->{unique},
7335             );
7336         }
7337
7338         $fh->close();
7339     };
7340
7341     eval {
7342         # enable interrupts
7343         local $SIG{INT} =
7344             local $SIG{TERM} =
7345             local $SIG{QUIT} =
7346             local $SIG{HUP} =
7347             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7348         local $SIG{ALRM} = sub { die "got timeout\n"; };
7349
7350         $oldtimeout = alarm($timeout);
7351
7352         my $parser = sub {
7353             my $line = shift;
7354
7355             print "$line\n";
7356
7357             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7358                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7359                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7360             } elsif ($line =~ m/^CTIME: /) {
7361                 # we correctly received the vma config, so we can disable
7362                 # the timeout now for disk allocation (set to 10 minutes, so
7363                 # that we always timeout if something goes wrong)
7364                 alarm(600);
7365                 &$print_devmap();
7366                 print $fifofh "done\n";
7367                 my $tmp = $oldtimeout || 0;
7368                 $oldtimeout = undef;
7369                 alarm($tmp);
7370                 close($fifofh);
7371                 $fifofh = undef;
7372             }
7373         };
7374
7375         print "restore vma archive: $dbg_cmdstring\n";
7376         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7377     };
7378     my $err = $@;
7379
7380     alarm($oldtimeout) if $oldtimeout;
7381
7382     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7383
7384     close($fifofh) if $fifofh;
7385     unlink $mapfifo;
7386     rmtree $tmpdir;
7387
7388     if ($err) {
7389         $restore_destroy_volumes->($cfg, $virtdev_hash);
7390         die $err;
7391     }
7392
7393     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7394     PVE::QemuConfig->write_config($vmid, $new_conf);
7395
7396     eval { rescan($vmid, 1); };
7397     warn $@ if $@;
7398
7399     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7400 }
7401
7402 sub restore_tar_archive {
7403     my ($archive, $vmid, $user, $opts) = @_;
7404
7405     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7406         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7407         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7408     }
7409
7410     if ($archive ne '-') {
7411         my $firstfile = tar_archive_read_firstfile($archive);
7412         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7413             if $firstfile ne 'qemu-server.conf';
7414     }
7415
7416     my $storecfg = PVE::Storage::config();
7417
7418     # avoid zombie disks when restoring over an existing VM -> cleanup first
7419     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7420     # skiplock=1 because qmrestore has set the 'create' lock itself already
7421     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7422     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7423
7424     my $tocmd = "/usr/lib/qemu-server/qmextract";
7425
7426     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7427     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7428     $tocmd .= ' --prealloc' if $opts->{prealloc};
7429     $tocmd .= ' --info' if $opts->{info};
7430
7431     # tar option "xf" does not autodetect compression when read from STDIN,
7432     # so we pipe to zcat
7433     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7434         PVE::Tools::shellquote("--to-command=$tocmd");
7435
7436     my $tmpdir = "/var/tmp/vzdumptmp$$";
7437     mkpath $tmpdir;
7438
7439     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7440     local $ENV{VZDUMP_VMID} = $vmid;
7441     local $ENV{VZDUMP_USER} = $user;
7442
7443     my $conffile = PVE::QemuConfig->config_file($vmid);
7444     my $new_conf_raw = '';
7445
7446     # disable interrupts (always do cleanups)
7447     local $SIG{INT} =
7448         local $SIG{TERM} =
7449         local $SIG{QUIT} =
7450         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7451
7452     eval {
7453         # enable interrupts
7454         local $SIG{INT} =
7455             local $SIG{TERM} =
7456             local $SIG{QUIT} =
7457             local $SIG{HUP} =
7458             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7459
7460         if ($archive eq '-') {
7461             print "extracting archive from STDIN\n";
7462             run_command($cmd, input => "<&STDIN");
7463         } else {
7464             print "extracting archive '$archive'\n";
7465             run_command($cmd);
7466         }
7467
7468         return if $opts->{info};
7469
7470         # read new mapping
7471         my $map = {};
7472         my $statfile = "$tmpdir/qmrestore.stat";
7473         if (my $fd = IO::File->new($statfile, "r")) {
7474             while (defined (my $line = <$fd>)) {
7475                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7476                     $map->{$1} = $2 if $1;
7477                 } else {
7478                     print STDERR "unable to parse line in statfile - $line\n";
7479                 }
7480             }
7481             $fd->close();
7482         }
7483
7484         my $confsrc = "$tmpdir/qemu-server.conf";
7485
7486         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7487
7488         my $cookie = { netcount => 0 };
7489         while (defined (my $line = <$srcfd>)) {
7490             $new_conf_raw .= restore_update_config_line(
7491                 $cookie,
7492                 $map,
7493                 $line,
7494                 $opts->{unique},
7495             );
7496         }
7497
7498         $srcfd->close();
7499     };
7500     if (my $err = $@) {
7501         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7502         die $err;
7503     }
7504
7505     rmtree $tmpdir;
7506
7507     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7508
7509     PVE::Cluster::cfs_update(); # make sure we read new file
7510
7511     eval { rescan($vmid, 1); };
7512     warn $@ if $@;
7513 };
7514
7515 sub foreach_storage_used_by_vm {
7516     my ($conf, $func) = @_;
7517
7518     my $sidhash = {};
7519
7520     PVE::QemuConfig->foreach_volume($conf, sub {
7521         my ($ds, $drive) = @_;
7522         return if drive_is_cdrom($drive);
7523
7524         my $volid = $drive->{file};
7525
7526         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7527         $sidhash->{$sid} = $sid if $sid;
7528     });
7529
7530     foreach my $sid (sort keys %$sidhash) {
7531         &$func($sid);
7532     }
7533 }
7534
7535 my $qemu_snap_storage = {
7536     rbd => 1,
7537 };
7538 sub do_snapshots_with_qemu {
7539     my ($storecfg, $volid, $deviceid) = @_;
7540
7541     return if $deviceid =~ m/tpmstate0/;
7542
7543     my $storage_name = PVE::Storage::parse_volume_id($volid);
7544     my $scfg = $storecfg->{ids}->{$storage_name};
7545     die "could not find storage '$storage_name'\n" if !defined($scfg);
7546
7547     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7548         return 1;
7549     }
7550
7551     if ($volid =~ m/\.(qcow2|qed)$/){
7552         return 1;
7553     }
7554
7555     return;
7556 }
7557
7558 sub qga_check_running {
7559     my ($vmid, $nowarn) = @_;
7560
7561     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7562     if ($@) {
7563         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7564         return 0;
7565     }
7566     return 1;
7567 }
7568
7569 sub template_create {
7570     my ($vmid, $conf, $disk) = @_;
7571
7572     my $storecfg = PVE::Storage::config();
7573
7574     PVE::QemuConfig->foreach_volume($conf, sub {
7575         my ($ds, $drive) = @_;
7576
7577         return if drive_is_cdrom($drive);
7578         return if $disk && $ds ne $disk;
7579
7580         my $volid = $drive->{file};
7581         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7582
7583         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7584         $drive->{file} = $voliddst;
7585         $conf->{$ds} = print_drive($drive);
7586         PVE::QemuConfig->write_config($vmid, $conf);
7587     });
7588 }
7589
7590 sub convert_iscsi_path {
7591     my ($path) = @_;
7592
7593     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7594         my $portal = $1;
7595         my $target = $2;
7596         my $lun = $3;
7597
7598         my $initiator_name = get_initiator_name();
7599
7600         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7601                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7602     }
7603
7604     die "cannot convert iscsi path '$path', unkown format\n";
7605 }
7606
7607 sub qemu_img_convert {
7608     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7609
7610     my $storecfg = PVE::Storage::config();
7611     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7612     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7613
7614     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7615
7616     my $cachemode;
7617     my $src_path;
7618     my $src_is_iscsi = 0;
7619     my $src_format;
7620
7621     if ($src_storeid) {
7622         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7623         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7624         $src_format = qemu_img_format($src_scfg, $src_volname);
7625         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7626         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7627         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7628     } elsif (-f $src_volid || -b $src_volid) {
7629         $src_path = $src_volid;
7630         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7631             $src_format = $1;
7632         }
7633     }
7634
7635     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7636
7637     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7638     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7639     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7640     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7641
7642     my $cmd = [];
7643     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7644     push @$cmd, '-l', "snapshot.name=$snapname"
7645         if $snapname && $src_format && $src_format eq "qcow2";
7646     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7647     push @$cmd, '-T', $cachemode if defined($cachemode);
7648
7649     if ($src_is_iscsi) {
7650         push @$cmd, '--image-opts';
7651         $src_path = convert_iscsi_path($src_path);
7652     } elsif ($src_format) {
7653         push @$cmd, '-f', $src_format;
7654     }
7655
7656     if ($dst_is_iscsi) {
7657         push @$cmd, '--target-image-opts';
7658         $dst_path = convert_iscsi_path($dst_path);
7659     } else {
7660         push @$cmd, '-O', $dst_format;
7661     }
7662
7663     push @$cmd, $src_path;
7664
7665     if (!$dst_is_iscsi && $is_zero_initialized) {
7666         push @$cmd, "zeroinit:$dst_path";
7667     } else {
7668         push @$cmd, $dst_path;
7669     }
7670
7671     my $parser = sub {
7672         my $line = shift;
7673         if($line =~ m/\((\S+)\/100\%\)/){
7674             my $percent = $1;
7675             my $transferred = int($size * $percent / 100);
7676             my $total_h = render_bytes($size, 1);
7677             my $transferred_h = render_bytes($transferred, 1);
7678
7679             print "transferred $transferred_h of $total_h ($percent%)\n";
7680         }
7681
7682     };
7683
7684     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7685     my $err = $@;
7686     die "copy failed: $err" if $err;
7687 }
7688
7689 sub qemu_img_format {
7690     my ($scfg, $volname) = @_;
7691
7692     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7693         return $1;
7694     } else {
7695         return "raw";
7696     }
7697 }
7698
7699 sub qemu_drive_mirror {
7700     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7701
7702     $jobs = {} if !$jobs;
7703
7704     my $qemu_target;
7705     my $format;
7706     $jobs->{"drive-$drive"} = {};
7707
7708     if ($dst_volid =~ /^nbd:/) {
7709         $qemu_target = $dst_volid;
7710         $format = "nbd";
7711     } else {
7712         my $storecfg = PVE::Storage::config();
7713         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7714
7715         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7716
7717         $format = qemu_img_format($dst_scfg, $dst_volname);
7718
7719         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7720
7721         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7722     }
7723
7724     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7725     $opts->{format} = $format if $format;
7726
7727     if (defined($src_bitmap)) {
7728         $opts->{sync} = 'incremental';
7729         $opts->{bitmap} = $src_bitmap;
7730         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7731     }
7732
7733     if (defined($bwlimit)) {
7734         $opts->{speed} = $bwlimit * 1024;
7735         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7736     } else {
7737         print "drive mirror is starting for drive-$drive\n";
7738     }
7739
7740     # if a job already runs for this device we get an error, catch it for cleanup
7741     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7742     if (my $err = $@) {
7743         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7744         warn "$@\n" if $@;
7745         die "mirroring error: $err\n";
7746     }
7747
7748     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7749 }
7750
7751 # $completion can be either
7752 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7753 # 'cancel': wait until all jobs are ready, block-job-cancel them
7754 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7755 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7756 sub qemu_drive_mirror_monitor {
7757     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7758
7759     $completion //= 'complete';
7760     $op //= "mirror";
7761
7762     eval {
7763         my $err_complete = 0;
7764
7765         my $starttime = time ();
7766         while (1) {
7767             die "block job ('$op') timed out\n" if $err_complete > 300;
7768
7769             my $stats = mon_cmd($vmid, "query-block-jobs");
7770             my $ctime = time();
7771
7772             my $running_jobs = {};
7773             for my $stat (@$stats) {
7774                 next if $stat->{type} ne $op;
7775                 $running_jobs->{$stat->{device}} = $stat;
7776             }
7777
7778             my $readycounter = 0;
7779
7780             for my $job_id (sort keys %$jobs) {
7781                 my $job = $running_jobs->{$job_id};
7782
7783                 my $vanished = !defined($job);
7784                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7785                 if($complete || ($vanished && $completion eq 'auto')) {
7786                     print "$job_id: $op-job finished\n";
7787                     delete $jobs->{$job_id};
7788                     next;
7789                 }
7790
7791                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7792
7793                 my $busy = $job->{busy};
7794                 my $ready = $job->{ready};
7795                 if (my $total = $job->{len}) {
7796                     my $transferred = $job->{offset} || 0;
7797                     my $remaining = $total - $transferred;
7798                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7799
7800                     my $duration = $ctime - $starttime;
7801                     my $total_h = render_bytes($total, 1);
7802                     my $transferred_h = render_bytes($transferred, 1);
7803
7804                     my $status = sprintf(
7805                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7806                         render_duration($duration),
7807                     );
7808
7809                     if ($ready) {
7810                         if ($busy) {
7811                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7812                         } else {
7813                             $status .= ", ready";
7814                         }
7815                     }
7816                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7817                     $jobs->{$job_id}->{ready} = $ready;
7818                 }
7819
7820                 $readycounter++ if $job->{ready};
7821             }
7822
7823             last if scalar(keys %$jobs) == 0;
7824
7825             if ($readycounter == scalar(keys %$jobs)) {
7826                 print "all '$op' jobs are ready\n";
7827
7828                 # do the complete later (or has already been done)
7829                 last if $completion eq 'skip' || $completion eq 'auto';
7830
7831                 if ($vmiddst && $vmiddst != $vmid) {
7832                     my $agent_running = $qga && qga_check_running($vmid);
7833                     if ($agent_running) {
7834                         print "freeze filesystem\n";
7835                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7836                         warn $@ if $@;
7837                     } else {
7838                         print "suspend vm\n";
7839                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7840                         warn $@ if $@;
7841                     }
7842
7843                     # if we clone a disk for a new target vm, we don't switch the disk
7844                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7845
7846                     if ($agent_running) {
7847                         print "unfreeze filesystem\n";
7848                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7849                         warn $@ if $@;
7850                     } else {
7851                         print "resume vm\n";
7852                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7853                         warn $@ if $@;
7854                     }
7855
7856                     last;
7857                 } else {
7858
7859                     for my $job_id (sort keys %$jobs) {
7860                         # try to switch the disk if source and destination are on the same guest
7861                         print "$job_id: Completing block job_id...\n";
7862
7863                         my $op;
7864                         if ($completion eq 'complete') {
7865                             $op = 'block-job-complete';
7866                         } elsif ($completion eq 'cancel') {
7867                             $op = 'block-job-cancel';
7868                         } else {
7869                             die "invalid completion value: $completion\n";
7870                         }
7871                         eval { mon_cmd($vmid, $op, device => $job_id) };
7872                         if ($@ =~ m/cannot be completed/) {
7873                             print "$job_id: block job cannot be completed, trying again.\n";
7874                             $err_complete++;
7875                         }else {
7876                             print "$job_id: Completed successfully.\n";
7877                             $jobs->{$job_id}->{complete} = 1;
7878                         }
7879                     }
7880                 }
7881             }
7882             sleep 1;
7883         }
7884     };
7885     my $err = $@;
7886
7887     if ($err) {
7888         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7889         die "block job ($op) error: $err";
7890     }
7891 }
7892
7893 sub qemu_blockjobs_cancel {
7894     my ($vmid, $jobs) = @_;
7895
7896     foreach my $job (keys %$jobs) {
7897         print "$job: Cancelling block job\n";
7898         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7899         $jobs->{$job}->{cancel} = 1;
7900     }
7901
7902     while (1) {
7903         my $stats = mon_cmd($vmid, "query-block-jobs");
7904
7905         my $running_jobs = {};
7906         foreach my $stat (@$stats) {
7907             $running_jobs->{$stat->{device}} = $stat;
7908         }
7909
7910         foreach my $job (keys %$jobs) {
7911
7912             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7913                 print "$job: Done.\n";
7914                 delete $jobs->{$job};
7915             }
7916         }
7917
7918         last if scalar(keys %$jobs) == 0;
7919
7920         sleep 1;
7921     }
7922 }
7923
7924 sub clone_disk {
7925     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7926
7927     my ($vmid, $running) = $source->@{qw(vmid running)};
7928     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7929
7930     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7931     my ($storage, $format) = $dest->@{qw(storage format)};
7932
7933     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7934
7935     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7936         die "cloning from/to EFI disk requires EFI disk\n"
7937             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7938         die "cloning from/to TPM state requires TPM state\n"
7939             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7940
7941         # This would lead to two device nodes in QEMU pointing to the same backing image!
7942         die "cannot change drive name when cloning disk from/to the same VM\n"
7943             if $use_drive_mirror && $vmid == $newvmid;
7944     }
7945
7946     die "cannot move TPM state while VM is running\n"
7947         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7948
7949     my $newvolid;
7950
7951     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7952     print "$src_drivename " if $src_drivename;
7953     print "($drive->{file})\n";
7954
7955     if (!$full) {
7956         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7957         push @$newvollist, $newvolid;
7958     } else {
7959
7960         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7961         $storeid = $storage if $storage;
7962
7963         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7964
7965         my $name = undef;
7966         my $size = undef;
7967         if (drive_is_cloudinit($drive)) {
7968             $name = "vm-$newvmid-cloudinit";
7969             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7970             if ($scfg->{path}) {
7971                 $name .= ".$dst_format";
7972             }
7973             $snapname = undef;
7974             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7975         } elsif ($dst_drivename eq 'efidisk0') {
7976             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7977         } elsif ($dst_drivename eq 'tpmstate0') {
7978             $dst_format = 'raw';
7979             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7980         } else {
7981             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7982         }
7983         $newvolid = PVE::Storage::vdisk_alloc(
7984             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7985         );
7986         push @$newvollist, $newvolid;
7987
7988         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7989
7990         if (drive_is_cloudinit($drive)) {
7991             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7992             # if this is the case, we have to complete any block-jobs still there from
7993             # previous drive-mirrors
7994             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7995                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7996             }
7997             goto no_data_clone;
7998         }
7999
8000         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8001         if ($use_drive_mirror) {
8002             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8003                 $completion, $qga, $bwlimit);
8004         } else {
8005             # TODO: handle bwlimits
8006             if ($dst_drivename eq 'efidisk0') {
8007                 # the relevant data on the efidisk may be smaller than the source
8008                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8009                 # that is given by the OVMF_VARS.fd
8010                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8011                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8012
8013                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8014
8015                 # better for Ceph if block size is not too small, see bug #3324
8016                 my $bs = 1024*1024;
8017
8018                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8019
8020                 if ($src_format eq 'qcow2' && $snapname) {
8021                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8022                         if !min_version(kvm_user_version(), 6, 2);
8023                     push $cmd->@*, '-l', $snapname;
8024                 }
8025                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8026                 run_command($cmd);
8027             } else {
8028                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
8029             }
8030         }
8031     }
8032
8033 no_data_clone:
8034     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8035
8036     my $disk = dclone($drive);
8037     delete $disk->{format};
8038     $disk->{file} = $newvolid;
8039     $disk->{size} = $size if defined($size);
8040
8041     return $disk;
8042 }
8043
8044 sub get_running_qemu_version {
8045     my ($vmid) = @_;
8046     my $res = mon_cmd($vmid, "query-version");
8047     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8048 }
8049
8050 sub qemu_use_old_bios_files {
8051     my ($machine_type) = @_;
8052
8053     return if !$machine_type;
8054
8055     my $use_old_bios_files = undef;
8056
8057     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8058         $machine_type = $1;
8059         $use_old_bios_files = 1;
8060     } else {
8061         my $version = extract_version($machine_type, kvm_user_version());
8062         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8063         # load new efi bios files on migration. So this hack is required to allow
8064         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8065         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8066         $use_old_bios_files = !min_version($version, 2, 4);
8067     }
8068
8069     return ($use_old_bios_files, $machine_type);
8070 }
8071
8072 sub get_efivars_size {
8073     my ($conf, $efidisk) = @_;
8074
8075     my $arch = get_vm_arch($conf);
8076     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8077     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8078     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8079     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
8080     return -s $ovmf_vars;
8081 }
8082
8083 sub update_efidisk_size {
8084     my ($conf) = @_;
8085
8086     return if !defined($conf->{efidisk0});
8087
8088     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8089     $disk->{size} = get_efivars_size($conf);
8090     $conf->{efidisk0} = print_drive($disk);
8091
8092     return;
8093 }
8094
8095 sub update_tpmstate_size {
8096     my ($conf) = @_;
8097
8098     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8099     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8100     $conf->{tpmstate0} = print_drive($disk);
8101 }
8102
8103 sub create_efidisk($$$$$$$) {
8104     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8105
8106     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8107     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
8108
8109     my $vars_size_b = -s $ovmf_vars;
8110     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8111     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8112     PVE::Storage::activate_volumes($storecfg, [$volid]);
8113
8114     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8115     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8116
8117     return ($volid, $size/1024);
8118 }
8119
8120 sub vm_iothreads_list {
8121     my ($vmid) = @_;
8122
8123     my $res = mon_cmd($vmid, 'query-iothreads');
8124
8125     my $iothreads = {};
8126     foreach my $iothread (@$res) {
8127         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8128     }
8129
8130     return $iothreads;
8131 }
8132
8133 sub scsihw_infos {
8134     my ($conf, $drive) = @_;
8135
8136     my $maxdev = 0;
8137
8138     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8139         $maxdev = 7;
8140     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8141         $maxdev = 1;
8142     } else {
8143         $maxdev = 256;
8144     }
8145
8146     my $controller = int($drive->{index} / $maxdev);
8147     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8148         ? "virtioscsi"
8149         : "scsihw";
8150
8151     return ($maxdev, $controller, $controller_prefix);
8152 }
8153
8154 sub resolve_dst_disk_format {
8155         my ($storecfg, $storeid, $src_volname, $format) = @_;
8156         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8157
8158         if (!$format) {
8159             # if no target format is specified, use the source disk format as hint
8160             if ($src_volname) {
8161                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8162                 $format = qemu_img_format($scfg, $src_volname);
8163             } else {
8164                 return $defFormat;
8165             }
8166         }
8167
8168         # test if requested format is supported - else use default
8169         my $supported = grep { $_ eq $format } @$validFormats;
8170         $format = $defFormat if !$supported;
8171         return $format;
8172 }
8173
8174 # NOTE: if this logic changes, please update docs & possibly gui logic
8175 sub find_vmstate_storage {
8176     my ($conf, $storecfg) = @_;
8177
8178     # first, return storage from conf if set
8179     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8180
8181     my ($target, $shared, $local);
8182
8183     foreach_storage_used_by_vm($conf, sub {
8184         my ($sid) = @_;
8185         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8186         my $dst = $scfg->{shared} ? \$shared : \$local;
8187         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8188     });
8189
8190     # second, use shared storage where VM has at least one disk
8191     # third, use local storage where VM has at least one disk
8192     # fall back to local storage
8193     $target = $shared // $local // 'local';
8194
8195     return $target;
8196 }
8197
8198 sub generate_uuid {
8199     my ($uuid, $uuid_str);
8200     UUID::generate($uuid);
8201     UUID::unparse($uuid, $uuid_str);
8202     return $uuid_str;
8203 }
8204
8205 sub generate_smbios1_uuid {
8206     return "uuid=".generate_uuid();
8207 }
8208
8209 sub nbd_stop {
8210     my ($vmid) = @_;
8211
8212     mon_cmd($vmid, 'nbd-server-stop');
8213 }
8214
8215 sub create_reboot_request {
8216     my ($vmid) = @_;
8217     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8218         or die "failed to create reboot trigger file: $!\n";
8219     close($fh);
8220 }
8221
8222 sub clear_reboot_request {
8223     my ($vmid) = @_;
8224     my $path = "/run/qemu-server/$vmid.reboot";
8225     my $res = 0;
8226
8227     $res = unlink($path);
8228     die "could not remove reboot request for $vmid: $!"
8229         if !$res && $! != POSIX::ENOENT;
8230
8231     return $res;
8232 }
8233
8234 sub bootorder_from_legacy {
8235     my ($conf, $bootcfg) = @_;
8236
8237     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8238     my $bootindex_hash = {};
8239     my $i = 1;
8240     foreach my $o (split(//, $boot)) {
8241         $bootindex_hash->{$o} = $i*100;
8242         $i++;
8243     }
8244
8245     my $bootorder = {};
8246
8247     PVE::QemuConfig->foreach_volume($conf, sub {
8248         my ($ds, $drive) = @_;
8249
8250         if (drive_is_cdrom ($drive, 1)) {
8251             if ($bootindex_hash->{d}) {
8252                 $bootorder->{$ds} = $bootindex_hash->{d};
8253                 $bootindex_hash->{d} += 1;
8254             }
8255         } elsif ($bootindex_hash->{c}) {
8256             $bootorder->{$ds} = $bootindex_hash->{c}
8257                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8258             $bootindex_hash->{c} += 1;
8259         }
8260     });
8261
8262     if ($bootindex_hash->{n}) {
8263         for (my $i = 0; $i < $MAX_NETS; $i++) {
8264             my $netname = "net$i";
8265             next if !$conf->{$netname};
8266             $bootorder->{$netname} = $bootindex_hash->{n};
8267             $bootindex_hash->{n} += 1;
8268         }
8269     }
8270
8271     return $bootorder;
8272 }
8273
8274 # Generate default device list for 'boot: order=' property. Matches legacy
8275 # default boot order, but with explicit device names. This is important, since
8276 # the fallback for when neither 'order' nor the old format is specified relies
8277 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8278 sub get_default_bootdevices {
8279     my ($conf) = @_;
8280
8281     my @ret = ();
8282
8283     # harddisk
8284     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8285     push @ret, $first if $first;
8286
8287     # cdrom
8288     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8289     push @ret, $first if $first;
8290
8291     # network
8292     for (my $i = 0; $i < $MAX_NETS; $i++) {
8293         my $netname = "net$i";
8294         next if !$conf->{$netname};
8295         push @ret, $netname;
8296         last;
8297     }
8298
8299     return \@ret;
8300 }
8301
8302 sub device_bootorder {
8303     my ($conf) = @_;
8304
8305     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8306
8307     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8308
8309     my $bootorder = {};
8310     if (!defined($boot) || $boot->{legacy}) {
8311         $bootorder = bootorder_from_legacy($conf, $boot);
8312     } elsif ($boot->{order}) {
8313         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8314         for my $dev (PVE::Tools::split_list($boot->{order})) {
8315             $bootorder->{$dev} = $i++;
8316         }
8317     }
8318
8319     return $bootorder;
8320 }
8321
8322 sub register_qmeventd_handle {
8323     my ($vmid) = @_;
8324
8325     my $fh;
8326     my $peer = "/var/run/qmeventd.sock";
8327     my $count = 0;
8328
8329     for (;;) {
8330         $count++;
8331         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8332         last if $fh;
8333         if ($! != EINTR && $! != EAGAIN) {
8334             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8335         }
8336         if ($count > 4) {
8337             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8338               . "after $count retries\n";
8339         }
8340         usleep(25000);
8341     }
8342
8343     # send handshake to mark VM as backing up
8344     print $fh to_json({vzdump => {vmid => "$vmid"}});
8345
8346     # return handle to be closed later when inhibit is no longer required
8347     return $fh;
8348 }
8349
8350 # bash completion helper
8351
8352 sub complete_backup_archives {
8353     my ($cmdname, $pname, $cvalue) = @_;
8354
8355     my $cfg = PVE::Storage::config();
8356
8357     my $storeid;
8358
8359     if ($cvalue =~ m/^([^:]+):/) {
8360         $storeid = $1;
8361     }
8362
8363     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8364
8365     my $res = [];
8366     foreach my $id (keys %$data) {
8367         foreach my $item (@{$data->{$id}}) {
8368             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8369             push @$res, $item->{volid} if defined($item->{volid});
8370         }
8371     }
8372
8373     return $res;
8374 }
8375
8376 my $complete_vmid_full = sub {
8377     my ($running) = @_;
8378
8379     my $idlist = vmstatus();
8380
8381     my $res = [];
8382
8383     foreach my $id (keys %$idlist) {
8384         my $d = $idlist->{$id};
8385         if (defined($running)) {
8386             next if $d->{template};
8387             next if $running && $d->{status} ne 'running';
8388             next if !$running && $d->{status} eq 'running';
8389         }
8390         push @$res, $id;
8391
8392     }
8393     return $res;
8394 };
8395
8396 sub complete_vmid {
8397     return &$complete_vmid_full();
8398 }
8399
8400 sub complete_vmid_stopped {
8401     return &$complete_vmid_full(0);
8402 }
8403
8404 sub complete_vmid_running {
8405     return &$complete_vmid_full(1);
8406 }
8407
8408 sub complete_storage {
8409
8410     my $cfg = PVE::Storage::config();
8411     my $ids = $cfg->{ids};
8412
8413     my $res = [];
8414     foreach my $sid (keys %$ids) {
8415         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8416         next if !$ids->{$sid}->{content}->{images};
8417         push @$res, $sid;
8418     }
8419
8420     return $res;
8421 }
8422
8423 sub complete_migration_storage {
8424     my ($cmd, $param, $current_value, $all_args) = @_;
8425
8426     my $targetnode = @$all_args[1];
8427
8428     my $cfg = PVE::Storage::config();
8429     my $ids = $cfg->{ids};
8430
8431     my $res = [];
8432     foreach my $sid (keys %$ids) {
8433         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8434         next if !$ids->{$sid}->{content}->{images};
8435         push @$res, $sid;
8436     }
8437
8438     return $res;
8439 }
8440
8441 sub vm_is_paused {
8442     my ($vmid) = @_;
8443     my $qmpstatus = eval {
8444         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8445         mon_cmd($vmid, "query-status");
8446     };
8447     warn "$@\n" if $@;
8448     return $qmpstatus && $qmpstatus->{status} eq "paused";
8449 }
8450
8451 sub check_volume_storage_type {
8452     my ($storecfg, $vol) = @_;
8453
8454     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8455     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8456     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8457
8458     die "storage '$storeid' does not support content-type '$vtype'\n"
8459         if !$scfg->{content}->{$vtype};
8460
8461     return 1;
8462 }
8463
8464 sub add_nets_bridge_fdb {
8465     my ($conf, $vmid) = @_;
8466
8467     for my $opt (keys %$conf) {
8468         next if $opt !~ m/^net(\d+)$/;
8469         my $iface = "tap${vmid}i$1";
8470         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8471         my $net = parse_net($conf->{$opt}, 1) or next;
8472
8473         my $mac = $net->{macaddr};
8474         if (!$mac) {
8475             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8476                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8477             next;
8478         }
8479
8480         my $bridge = $net->{bridge};
8481         if ($have_sdn) {
8482             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8483         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8484             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8485         }
8486     }
8487 }
8488
8489 sub del_nets_bridge_fdb {
8490     my ($conf, $vmid) = @_;
8491
8492     for my $opt (keys %$conf) {
8493         next if $opt !~ m/^net(\d+)$/;
8494         my $iface = "tap${vmid}i$1";
8495
8496         my $net = parse_net($conf->{$opt}) or next;
8497         my $mac = $net->{macaddr} or next;
8498
8499         my $bridge = $net->{bridge};
8500         if ($have_sdn) {
8501             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8502         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8503             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8504         }
8505     }
8506 }
8507
8508 1;