PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory;
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::USB;
  62
  63 my $have_sdn;
  64 eval {
  65     require PVE::Network::SDN::Zones;
  66     $have_sdn = 1;
  67 };
  68
  69 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  70 my $OVMF = {
  71     x86_64 => {
  72         '4m-no-smm' => [
  73             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  74             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  75         ],
  76         '4m-no-smm-ms' => [
  77             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  78             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  79         ],
  80         '4m' => [
  81             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  82             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  83         ],
  84         '4m-ms' => [
  85             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  86             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  87         ],
  88         default => [
  89             "$EDK2_FW_BASE/OVMF_CODE.fd",
  90             "$EDK2_FW_BASE/OVMF_VARS.fd",
  91         ],
  92     },
  93     aarch64 => {
  94         default => [
  95             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  96             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  97         ],
  98     },
  99 };
 100
 101 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 102
 103 # Note about locking: we use flock on the config file protect against concurent actions.
 104 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 105 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 106 # But you can ignore this kind of lock with the --skiplock flag.
 107
 108 cfs_register_file(
 109     '/qemu-server/',
 110     \&parse_vm_config,
 111     \&write_vm_config
 112 );
 113
 114 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 115     description => "Some command save/restore state from this location.",
 116     type => 'string',
 117     maxLength => 128,
 118     optional => 1,
 119 });
 120
 121 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 122         description => "Specifies the QEMU machine type.",
 123         type => 'string',
 124         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 125         maxLength => 40,
 126         optional => 1,
 127 });
 128
 129 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 130 my $nodename_cache;
 131 sub nodename {
 132     $nodename_cache //= PVE::INotify::nodename();
 133     return $nodename_cache;
 134 }
 135
 136 my $watchdog_fmt = {
 137     model => {
 138         default_key => 1,
 139         type => 'string',
 140         enum => [qw(i6300esb ib700)],
 141         description => "Watchdog type to emulate.",
 142         default => 'i6300esb',
 143         optional => 1,
 144     },
 145     action => {
 146         type => 'string',
 147         enum => [qw(reset shutdown poweroff pause debug none)],
 148         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 149         optional => 1,
 150     },
 151 };
 152 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 153
 154 my $agent_fmt = {
 155     enabled => {
 156         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 157         type => 'boolean',
 158         default => 0,
 159         default_key => 1,
 160     },
 161     fstrim_cloned_disks => {
 162         description => "Run fstrim after moving a disk or migrating the VM.",
 163         type => 'boolean',
 164         optional => 1,
 165         default => 0,
 166     },
 167     'freeze-fs-on-backup' => {
 168         description => "Freeze/thaw guest filesystems on backup for consistency.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 1,
 172     },
 173     type => {
 174         description => "Select the agent type",
 175         type => 'string',
 176         default => 'virtio',
 177         optional => 1,
 178         enum => [qw(virtio isa)],
 179     },
 180 };
 181
 182 my $vga_fmt = {
 183     type => {
 184         description => "Select the VGA type.",
 185         type => 'string',
 186         default => 'std',
 187         optional => 1,
 188         default_key => 1,
 189         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 190     },
 191     memory => {
 192         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 193         type => 'integer',
 194         optional => 1,
 195         minimum => 4,
 196         maximum => 512,
 197     },
 198 };
 199
 200 my $ivshmem_fmt = {
 201     size => {
 202         type => 'integer',
 203         minimum => 1,
 204         description => "The size of the file in MB.",
 205     },
 206     name => {
 207         type => 'string',
 208         pattern => '[a-zA-Z0-9\-]+',
 209         optional => 1,
 210         format_description => 'string',
 211         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 212     },
 213 };
 214
 215 my $audio_fmt = {
 216     device => {
 217         type => 'string',
 218         enum => [qw(ich9-intel-hda intel-hda AC97)],
 219         description =>  "Configure an audio device."
 220     },
 221     driver =>  {
 222         type => 'string',
 223         enum => ['spice', 'none'],
 224         default => 'spice',
 225         optional => 1,
 226         description => "Driver backend for the audio device."
 227     },
 228 };
 229
 230 my $spice_enhancements_fmt = {
 231     foldersharing => {
 232         type => 'boolean',
 233         optional => 1,
 234         default => '0',
 235         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 236     },
 237     videostreaming =>  {
 238         type => 'string',
 239         enum => ['off', 'all', 'filter'],
 240         default => 'off',
 241         optional => 1,
 242         description => "Enable video streaming. Uses compression for detected video streams."
 243     },
 244 };
 245
 246 my $rng_fmt = {
 247     source => {
 248         type => 'string',
 249         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 250         default_key => 1,
 251         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 252             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 253             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 254             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 255             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 256             ." a hardware RNG from the host.",
 257     },
 258     max_bytes => {
 259         type => 'integer',
 260         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 261             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 262             ." `0` to disable limiting (potentially dangerous!).",
 263         optional => 1,
 264
 265         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 266         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 267         # reading from /dev/urandom
 268         default => 1024,
 269     },
 270     period => {
 271         type => 'integer',
 272         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 273             ." the guest to retrieve another 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $meta_info_fmt = {
 280     'ctime' => {
 281         type => 'integer',
 282         description => "The guest creation timestamp as UNIX epoch time",
 283         minimum => 0,
 284         optional => 1,
 285     },
 286     'creation-qemu' => {
 287         type => 'string',
 288         description => "The QEMU (machine) version from the time this VM was created.",
 289         pattern => '\d+(\.\d+)+',
 290         optional => 1,
 291     },
 292 };
 293
 294 my $confdesc = {
 295     onboot => {
 296         optional => 1,
 297         type => 'boolean',
 298         description => "Specifies whether a VM will be started during system bootup.",
 299         default => 0,
 300     },
 301     autostart => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Automatic restart after crash (currently ignored).",
 305         default => 0,
 306     },
 307     hotplug => {
 308         optional => 1,
 309         type => 'string', format => 'pve-hotplug-features',
 310         description => "Selectively enable hotplug features. This is a comma separated list of"
 311             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 312             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 313             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 314             ." windows > 7.",
 315         default => 'network,disk,usb',
 316     },
 317     reboot => {
 318         optional => 1,
 319         type => 'boolean',
 320         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 321         default => 1,
 322     },
 323     lock => {
 324         optional => 1,
 325         type => 'string',
 326         description => "Lock/unlock the VM.",
 327         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 328     },
 329     cpulimit => {
 330         optional => 1,
 331         type => 'number',
 332         description => "Limit of CPU usage.",
 333         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 334             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 335         minimum => 0,
 336         maximum => 128,
 337         default => 0,
 338     },
 339     cpuunits => {
 340         optional => 1,
 341         type => 'integer',
 342         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 343         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 344             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 345             ." weights of all the other running VMs.",
 346         minimum => 1,
 347         maximum => 262144,
 348         default => 'cgroup v1: 1024, cgroup v2: 100',
 349     },
 350     memory => {
 351         optional => 1,
 352         type => 'integer',
 353         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 354             ." you use the balloon device.",
 355         minimum => 16,
 356         default => 512,
 357     },
 358     balloon => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 362         minimum => 0,
 363     },
 364     shares => {
 365         optional => 1,
 366         type => 'integer',
 367         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 368             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 369             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 370         minimum => 0,
 371         maximum => 50000,
 372         default => 1000,
 373     },
 374     keyboard => {
 375         optional => 1,
 376         type => 'string',
 377         description => "Keyboard layout for VNC server. This option is generally not required and"
 378         ." is often better handled from within the guest OS.",
 379         enum => PVE::Tools::kvmkeymaplist(),
 380         default => undef,
 381     },
 382     name => {
 383         optional => 1,
 384         type => 'string', format => 'dns-name',
 385         description => "Set a name for the VM. Only used on the configuration web interface.",
 386     },
 387     scsihw => {
 388         optional => 1,
 389         type => 'string',
 390         description => "SCSI controller model",
 391         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 392         default => 'lsi',
 393     },
 394     description => {
 395         optional => 1,
 396         type => 'string',
 397         description => "Description for the VM. Shown in the web-interface VM's summary."
 398             ." This is saved as comment inside the configuration file.",
 399         maxLength => 1024 * 8,
 400     },
 401     ostype => {
 402         optional => 1,
 403         type => 'string',
 404         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 405         description => "Specify guest operating system.",
 406         verbose_description => <<EODESC,
 407 Specify guest operating system. This is used to enable special
 408 optimization/features for specific operating systems:
 409
 410 [horizontal]
 411 other;; unspecified OS
 412 wxp;; Microsoft Windows XP
 413 w2k;; Microsoft Windows 2000
 414 w2k3;; Microsoft Windows 2003
 415 w2k8;; Microsoft Windows 2008
 416 wvista;; Microsoft Windows Vista
 417 win7;; Microsoft Windows 7
 418 win8;; Microsoft Windows 8/2012/2012r2
 419 win10;; Microsoft Windows 10/2016/2019
 420 win11;; Microsoft Windows 11/2022
 421 l24;; Linux 2.4 Kernel
 422 l26;; Linux 2.6 - 6.X Kernel
 423 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 424 EODESC
 425     },
 426     boot => {
 427         optional => 1,
 428         type => 'string', format => 'pve-qm-boot',
 429         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 430             ." key or 'legacy=' is deprecated.",
 431     },
 432     bootdisk => {
 433         optional => 1,
 434         type => 'string', format => 'pve-qm-bootdisk',
 435         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 436         pattern => '(ide|sata|scsi|virtio)\d+',
 437     },
 438     smp => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPUs. Please use option -sockets instead.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     sockets => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of CPU sockets.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     cores => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of cores per socket.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     numa => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Enable/disable NUMA.",
 463         default => 0,
 464     },
 465     hugepages => {
 466         optional => 1,
 467         type => 'string',
 468         description => "Enable/disable hugepages memory.",
 469         enum => [qw(any 2 1024)],
 470     },
 471     keephugepages => {
 472         optional => 1,
 473         type => 'boolean',
 474         default => 0,
 475         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 476             ." after VM shutdown and can be used for subsequent starts.",
 477     },
 478     vcpus => {
 479         optional => 1,
 480         type => 'integer',
 481         description => "Number of hotplugged vcpus.",
 482         minimum => 1,
 483         default => 0,
 484     },
 485     acpi => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Enable/disable ACPI.",
 489         default => 1,
 490     },
 491     agent => {
 492         optional => 1,
 493         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 494         type => 'string',
 495         format => $agent_fmt,
 496     },
 497     kvm => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Enable/disable KVM hardware virtualization.",
 501         default => 1,
 502     },
 503     tdf => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable time drift fix.",
 507         default => 0,
 508     },
 509     localtime => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 513             ." the `ostype` indicates a Microsoft Windows OS.",
 514     },
 515     freeze => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 519     },
 520     vga => {
 521         optional => 1,
 522         type => 'string', format => $vga_fmt,
 523         description => "Configure the VGA hardware.",
 524         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 525             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 526             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 527             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 528             ." display server. For win* OS you can select how many independent displays you want,"
 529             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 530             ." using a serial device as terminal.",
 531     },
 532     watchdog => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-watchdog',
 535         description => "Create a virtual hardware watchdog device.",
 536         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 537             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 538             ." else the watchdog will reset the guest (or execute the respective action specified)",
 539     },
 540     startdate => {
 541         optional => 1,
 542         type => 'string',
 543         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 544         description => "Set the initial date of the real time clock. Valid format for date are:"
 545             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 546         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 547         default => 'now',
 548     },
 549     startup =>  get_standard_option('pve-startup-order'),
 550     template => {
 551         optional => 1,
 552         type => 'boolean',
 553         description => "Enable/disable Template.",
 554         default => 0,
 555     },
 556     args => {
 557         optional => 1,
 558         type => 'string',
 559         description => "Arbitrary arguments passed to kvm.",
 560         verbose_description => <<EODESCR,
 561 Arbitrary arguments passed to kvm, for example:
 562
 563 args: -no-reboot -smbios 'type=0,vendor=FOO'
 564
 565 NOTE: this option is for experts only.
 566 EODESCR
 567     },
 568     tablet => {
 569         optional => 1,
 570         type => 'boolean',
 571         default => 1,
 572         description => "Enable/disable the USB tablet device.",
 573         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 574             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 575             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 576             ." may consider disabling this to save some context switches. This is turned off by"
 577             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 578     },
 579     migrate_speed => {
 580         optional => 1,
 581         type => 'integer',
 582         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 583         minimum => 0,
 584         default => 0,
 585     },
 586     migrate_downtime => {
 587         optional => 1,
 588         type => 'number',
 589         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 590         minimum => 0,
 591         default => 0.1,
 592     },
 593     cdrom => {
 594         optional => 1,
 595         type => 'string', format => 'pve-qm-ide',
 596         typetext => '<volume>',
 597         description => "This is an alias for option -ide2",
 598     },
 599     cpu => {
 600         optional => 1,
 601         description => "Emulated CPU type.",
 602         type => 'string',
 603         format => 'pve-vm-cpu-conf',
 604     },
 605     parent => get_standard_option('pve-snapshot-name', {
 606         optional => 1,
 607         description => "Parent snapshot name. This is used internally, and should not be modified.",
 608     }),
 609     snaptime => {
 610         optional => 1,
 611         description => "Timestamp for snapshots.",
 612         type => 'integer',
 613         minimum => 0,
 614     },
 615     vmstate => {
 616         optional => 1,
 617         type => 'string', format => 'pve-volume-id',
 618         description => "Reference to a volume which stores the VM state. This is used internally"
 619             ." for snapshots.",
 620     },
 621     vmstatestorage => get_standard_option('pve-storage-id', {
 622         description => "Default storage for VM state volumes/files.",
 623         optional => 1,
 624     }),
 625     runningmachine => get_standard_option('pve-qemu-machine', {
 626         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 627             ." for snapshots.",
 628     }),
 629     runningcpu => {
 630         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 631             ." internally for snapshots.",
 632         optional => 1,
 633         type => 'string',
 634         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 635         format_description => 'QEMU -cpu parameter'
 636     },
 637     machine => get_standard_option('pve-qemu-machine'),
 638     arch => {
 639         description => "Virtual processor architecture. Defaults to the host.",
 640         optional => 1,
 641         type => 'string',
 642         enum => [qw(x86_64 aarch64)],
 643     },
 644     smbios1 => {
 645         description => "Specify SMBIOS type 1 fields.",
 646         type => 'string', format => 'pve-qm-smbios1',
 647         maxLength => 512,
 648         optional => 1,
 649     },
 650     protection => {
 651         optional => 1,
 652         type => 'boolean',
 653         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 654             ." remove disk operations.",
 655         default => 0,
 656     },
 657     bios => {
 658         optional => 1,
 659         type => 'string',
 660         enum => [ qw(seabios ovmf) ],
 661         description => "Select BIOS implementation.",
 662         default => 'seabios',
 663     },
 664     vmgenid => {
 665         type => 'string',
 666         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 667         format_description => 'UUID',
 668         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 669             ." to disable explicitly.",
 670         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 671             ." value identifier to the guest OS. This allows to notify the guest operating system"
 672             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 673             ." execution or creation from a template). The guest operating system notices the"
 674             ." change, and is then able to react as appropriate by marking its copies of"
 675             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 676             ."Note that auto-creation only works when done through API/CLI create or update methods"
 677             .", but not when manually editing the config file.",
 678         default => "1 (autogenerated)",
 679         optional => 1,
 680     },
 681     hookscript => {
 682         type => 'string',
 683         format => 'pve-volume-id',
 684         optional => 1,
 685         description => "Script that will be executed during various steps in the vms lifetime.",
 686     },
 687     ivshmem => {
 688         type => 'string',
 689         format => $ivshmem_fmt,
 690         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 691             ." the host.",
 692         optional => 1,
 693     },
 694     audio0 => {
 695         type => 'string',
 696         format => $audio_fmt,
 697         description => "Configure a audio device, useful in combination with QXL/Spice.",
 698         optional => 1
 699     },
 700     spice_enhancements => {
 701         type => 'string',
 702         format => $spice_enhancements_fmt,
 703         description => "Configure additional enhancements for SPICE.",
 704         optional => 1
 705     },
 706     tags => {
 707         type => 'string', format => 'pve-tag-list',
 708         description => 'Tags of the VM. This is only meta information.',
 709         optional => 1,
 710     },
 711     rng0 => {
 712         type => 'string',
 713         format => $rng_fmt,
 714         description => "Configure a VirtIO-based Random Number Generator.",
 715         optional => 1,
 716     },
 717     meta => {
 718         type => 'string',
 719         format => $meta_info_fmt,
 720         description => "Some (read-only) meta-information about this guest.",
 721         optional => 1,
 722     },
 723     affinity => {
 724         type => 'string', format => 'pve-cpuset',
 725         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 726         optional => 1,
 727     },
 728 };
 729
 730 my $cicustom_fmt = {
 731     meta => {
 732         type => 'string',
 733         optional => 1,
 734         description => 'Specify a custom file containing all meta data passed to the VM via"
 735             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 736         format => 'pve-volume-id',
 737         format_description => 'volume',
 738     },
 739     network => {
 740         type => 'string',
 741         optional => 1,
 742         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     user => {
 747         type => 'string',
 748         optional => 1,
 749         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 750         format => 'pve-volume-id',
 751         format_description => 'volume',
 752     },
 753     vendor => {
 754         type => 'string',
 755         optional => 1,
 756         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 757         format => 'pve-volume-id',
 758         format_description => 'volume',
 759     },
 760 };
 761 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 762
 763 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 764 my $confdesc_cloudinit = {
 765     citype => {
 766         optional => 1,
 767         type => 'string',
 768         description => 'Specifies the cloud-init configuration format. The default depends on the'
 769             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 770             .' and `configdrive2` for windows.',
 771         enum => ['configdrive2', 'nocloud', 'opennebula'],
 772     },
 773     ciuser => {
 774         optional => 1,
 775         type => 'string',
 776         description => "cloud-init: User name to change ssh keys and password for instead of the"
 777             ." image's configured default user.",
 778     },
 779     cipassword => {
 780         optional => 1,
 781         type => 'string',
 782         description => 'cloud-init: Password to assign the user. Using this is generally not'
 783             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 784             .' support hashed passwords.',
 785     },
 786     ciupgrade => {
 787         optional => 1,
 788         type => 'boolean',
 789         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 790         default => 1,
 791     },
 792     cicustom => {
 793         optional => 1,
 794         type => 'string',
 795         description => 'cloud-init: Specify custom files to replace the automatically generated'
 796             .' ones at start.',
 797         format => 'pve-qm-cicustom',
 798     },
 799     searchdomain => {
 800         optional => 1,
 801         type => 'string',
 802         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 803             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 804             .' are set.',
 805     },
 806     nameserver => {
 807         optional => 1,
 808         type => 'string', format => 'address-list',
 809         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 810             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 811             .' are set.',
 812     },
 813     sshkeys => {
 814         optional => 1,
 815         type => 'string',
 816         format => 'urlencoded',
 817         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 818     },
 819 };
 820
 821 # what about other qemu settings ?
 822 #cpu => 'string',
 823 #machine => 'string',
 824 #fda => 'file',
 825 #fdb => 'file',
 826 #mtdblock => 'file',
 827 #sd => 'file',
 828 #pflash => 'file',
 829 #snapshot => 'bool',
 830 #bootp => 'file',
 831 ##tftp => 'dir',
 832 ##smb => 'dir',
 833 #kernel => 'file',
 834 #append => 'string',
 835 #initrd => 'file',
 836 ##soundhw => 'string',
 837
 838 while (my ($k, $v) = each %$confdesc) {
 839     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 840 }
 841
 842 my $MAX_NETS = 32;
 843 my $MAX_SERIAL_PORTS = 4;
 844 my $MAX_PARALLEL_PORTS = 3;
 845 my $MAX_NUMA = 8;
 846
 847 my $numa_fmt = {
 848     cpus => {
 849         type => "string",
 850         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 851         description => "CPUs accessing this NUMA node.",
 852         format_description => "id[-id];...",
 853     },
 854     memory => {
 855         type => "number",
 856         description => "Amount of memory this NUMA node provides.",
 857         optional => 1,
 858     },
 859     hostnodes => {
 860         type => "string",
 861         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 862         description => "Host NUMA nodes to use.",
 863         format_description => "id[-id];...",
 864         optional => 1,
 865     },
 866     policy => {
 867         type => 'string',
 868         enum => [qw(preferred bind interleave)],
 869         description => "NUMA allocation policy.",
 870         optional => 1,
 871     },
 872 };
 873 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 874 my $numadesc = {
 875     optional => 1,
 876     type => 'string', format => $numa_fmt,
 877     description => "NUMA topology.",
 878 };
 879 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 880
 881 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 882     $confdesc->{"numa$i"} = $numadesc;
 883 }
 884
 885 my $nic_model_list = [
 886     'e1000',
 887     'e1000-82540em',
 888     'e1000-82544gc',
 889     'e1000-82545em',
 890     'e1000e',
 891     'i82551',
 892     'i82557b',
 893     'i82559er',
 894     'ne2k_isa',
 895     'ne2k_pci',
 896     'pcnet',
 897     'rtl8139',
 898     'virtio',
 899     'vmxnet3',
 900 ];
 901 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 902
 903 my $net_fmt_bridge_descr = <<__EOD__;
 904 Bridge to attach the network device to. The Proxmox VE standard bridge
 905 is called 'vmbr0'.
 906
 907 If you do not specify a bridge, we create a kvm user (NATed) network
 908 device, which provides DHCP and DNS services. The following addresses
 909 are used:
 910
 911  10.0.2.2   Gateway
 912  10.0.2.3   DNS Server
 913  10.0.2.4   SMB Server
 914
 915 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 916 __EOD__
 917
 918 my $net_fmt = {
 919     macaddr  => get_standard_option('mac-addr', {
 920         description => "MAC address. That address must be unique withing your network. This is"
 921             ." automatically generated if not specified.",
 922     }),
 923     model => {
 924         type => 'string',
 925         description => "Network Card Model. The 'virtio' model provides the best performance with"
 926             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 927             ." best to use 'e1000'.",
 928         enum => $nic_model_list,
 929         default_key => 1,
 930     },
 931     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 932     bridge => get_standard_option('pve-bridge-id', {
 933         description => $net_fmt_bridge_descr,
 934         optional => 1,
 935     }),
 936     queues => {
 937         type => 'integer',
 938         minimum => 0, maximum => 64,
 939         description => 'Number of packet queues to be used on the device.',
 940         optional => 1,
 941     },
 942     rate => {
 943         type => 'number',
 944         minimum => 0,
 945         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 946         optional => 1,
 947     },
 948     tag => {
 949         type => 'integer',
 950         minimum => 1, maximum => 4094,
 951         description => 'VLAN tag to apply to packets on this interface.',
 952         optional => 1,
 953     },
 954     trunks => {
 955         type => 'string',
 956         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 957         description => 'VLAN trunks to pass through this interface.',
 958         format_description => 'vlanid[;vlanid...]',
 959         optional => 1,
 960     },
 961     firewall => {
 962         type => 'boolean',
 963         description => 'Whether this interface should be protected by the firewall.',
 964         optional => 1,
 965     },
 966     link_down => {
 967         type => 'boolean',
 968         description => 'Whether this interface should be disconnected (like pulling the plug).',
 969         optional => 1,
 970     },
 971     mtu => {
 972         type => 'integer',
 973         minimum => 1, maximum => 65520,
 974         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 975         optional => 1,
 976     },
 977 };
 978
 979 my $netdesc = {
 980     optional => 1,
 981     type => 'string', format => $net_fmt,
 982     description => "Specify network devices.",
 983 };
 984
 985 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 986
 987 my $ipconfig_fmt = {
 988     ip => {
 989         type => 'string',
 990         format => 'pve-ipv4-config',
 991         format_description => 'IPv4Format/CIDR',
 992         description => 'IPv4 address in CIDR format.',
 993         optional => 1,
 994         default => 'dhcp',
 995     },
 996     gw => {
 997         type => 'string',
 998         format => 'ipv4',
 999         format_description => 'GatewayIPv4',
1000         description => 'Default gateway for IPv4 traffic.',
1001         optional => 1,
1002         requires => 'ip',
1003     },
1004     ip6 => {
1005         type => 'string',
1006         format => 'pve-ipv6-config',
1007         format_description => 'IPv6Format/CIDR',
1008         description => 'IPv6 address in CIDR format.',
1009         optional => 1,
1010         default => 'dhcp',
1011     },
1012     gw6 => {
1013         type => 'string',
1014         format => 'ipv6',
1015         format_description => 'GatewayIPv6',
1016         description => 'Default gateway for IPv6 traffic.',
1017         optional => 1,
1018         requires => 'ip6',
1019     },
1020 };
1021 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1022 my $ipconfigdesc = {
1023     optional => 1,
1024     type => 'string', format => 'pve-qm-ipconfig',
1025     description => <<'EODESCR',
1026 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1027
1028 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1029
1030 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1031 gateway should be provided.
1032 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1033 cloud-init 19.4 or newer.
1034
1035 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1036 dhcp on IPv4.
1037 EODESCR
1038 };
1039 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1040
1041 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1042     $confdesc->{"net$i"} = $netdesc;
1043     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1044 }
1045
1046 foreach my $key (keys %$confdesc_cloudinit) {
1047     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1048 }
1049
1050 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1051 sub pve_verify_cpuset {
1052     my ($set_text, $noerr) = @_;
1053
1054     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1055
1056     if ($@) {
1057         return if $noerr;
1058         die "unable to parse cpuset option\n";
1059     }
1060
1061     return PVE::CpuSet->new($members)->short_string();
1062 }
1063
1064 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1065 sub verify_volume_id_or_qm_path {
1066     my ($volid, $noerr) = @_;
1067
1068     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1069
1070     return verify_volume_id_or_absolute_path($volid, $noerr);
1071 }
1072
1073 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1074 sub verify_volume_id_or_absolute_path {
1075     my ($volid, $noerr) = @_;
1076
1077     return $volid if $volid =~ m|^/|;
1078
1079     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1080     if ($@) {
1081         return if $noerr;
1082         die $@;
1083     }
1084     return $volid;
1085 }
1086
1087 my $serialdesc = {
1088         optional => 1,
1089         type => 'string',
1090         pattern => '(/dev/.+|socket)',
1091         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1092         verbose_description =>  <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1096
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1098 use with special care.
1099
1100 CAUTION: Experimental! User reported problems with this option.
1101 EODESCR
1102 };
1103
1104 my $paralleldesc= {
1105         optional => 1,
1106         type => 'string',
1107         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1108         description =>  "Map host parallel devices (n is 0 to 2).",
1109         verbose_description =>  <<EODESCR,
1110 Map host parallel devices (n is 0 to 2).
1111
1112 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1113 machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1129 }
1130
1131 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1132     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1133 }
1134
1135 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1136     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1137 }
1138
1139 my $boot_fmt = {
1140     legacy => {
1141         optional => 1,
1142         default_key => 1,
1143         type => 'string',
1144         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1145                      . " Deprecated, use 'order=' instead.",
1146         pattern => '[acdn]{1,4}',
1147         format_description => "[acdn]{1,4}",
1148
1149         # note: this is also the fallback if boot: is not given at all
1150         default => 'cdn',
1151     },
1152     order => {
1153         optional => 1,
1154         type => 'string',
1155         format => 'pve-qm-bootdev-list',
1156         format_description => "device[;device...]",
1157         description => <<EODESC,
1158 The guest will attempt to boot from devices in the order they appear here.
1159
1160 Disks, optical drives and passed-through storage USB devices will be directly
1161 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1162 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1163
1164 Note that only devices in this list will be marked as bootable and thus loaded
1165 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1166 (e.g. software-raid), you need to specify all of them here.
1167
1168 Overrides the deprecated 'legacy=[acdn]*' value when given.
1169 EODESC
1170     },
1171 };
1172 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1173
1174 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1175 sub verify_bootdev {
1176     my ($dev, $noerr) = @_;
1177
1178     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1179     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1180
1181     my $check = sub {
1182         my ($base) = @_;
1183         return 0 if $dev !~ m/^$base\d+$/;
1184         return 0 if !$confdesc->{$dev};
1185         return 1;
1186     };
1187
1188     return $dev if $check->("net");
1189     return $dev if $check->("usb");
1190     return $dev if $check->("hostpci");
1191
1192     return if $noerr;
1193     die "invalid boot device '$dev'\n";
1194 }
1195
1196 sub print_bootorder {
1197     my ($devs) = @_;
1198     return "" if !@$devs;
1199     my $data = { order => join(';', @$devs) };
1200     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1201 }
1202
1203 my $kvm_api_version = 0;
1204
1205 sub kvm_version {
1206     return $kvm_api_version if $kvm_api_version;
1207
1208     open my $fh, '<', '/dev/kvm' or return;
1209
1210     # 0xae00 => KVM_GET_API_VERSION
1211     $kvm_api_version = ioctl($fh, 0xae00, 0);
1212     close($fh);
1213
1214     return $kvm_api_version;
1215 }
1216
1217 my $kvm_user_version = {};
1218 my $kvm_mtime = {};
1219
1220 sub kvm_user_version {
1221     my ($binary) = @_;
1222
1223     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1224     my $st = stat($binary);
1225
1226     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1227     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1228         $cachedmtime == $st->mtime;
1229
1230     $kvm_user_version->{$binary} = 'unknown';
1231     $kvm_mtime->{$binary} = $st->mtime;
1232
1233     my $code = sub {
1234         my $line = shift;
1235         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1236             $kvm_user_version->{$binary} = $2;
1237         }
1238     };
1239
1240     eval { run_command([$binary, '--version'], outfunc => $code); };
1241     warn $@ if $@;
1242
1243     return $kvm_user_version->{$binary};
1244
1245 }
1246 my sub extract_version {
1247     my ($machine_type, $version) = @_;
1248     $version = kvm_user_version() if !defined($version);
1249     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1250 }
1251
1252 sub kernel_has_vhost_net {
1253     return -c '/dev/vhost-net';
1254 }
1255
1256 sub option_exists {
1257     my $key = shift;
1258     return defined($confdesc->{$key});
1259 }
1260
1261 my $cdrom_path;
1262 sub get_cdrom_path {
1263
1264     return $cdrom_path if defined($cdrom_path);
1265
1266     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1267
1268     if (!defined($cdrom_path)) {
1269         log_warn("no physical CD-ROM available, ignoring");
1270         $cdrom_path = '';
1271     }
1272
1273     return $cdrom_path;
1274 }
1275
1276 sub get_iso_path {
1277     my ($storecfg, $vmid, $cdrom) = @_;
1278
1279     if ($cdrom eq 'cdrom') {
1280         return get_cdrom_path();
1281     } elsif ($cdrom eq 'none') {
1282         return '';
1283     } elsif ($cdrom =~ m|^/|) {
1284         return $cdrom;
1285     } else {
1286         return PVE::Storage::path($storecfg, $cdrom);
1287     }
1288 }
1289
1290 # try to convert old style file names to volume IDs
1291 sub filename_to_volume_id {
1292     my ($vmid, $file, $media) = @_;
1293
1294      if (!($file eq 'none' || $file eq 'cdrom' ||
1295           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1296
1297         return if $file =~ m|/|;
1298
1299         if ($media && $media eq 'cdrom') {
1300             $file = "local:iso/$file";
1301         } else {
1302             $file = "local:$vmid/$file";
1303         }
1304     }
1305
1306     return $file;
1307 }
1308
1309 sub verify_media_type {
1310     my ($opt, $vtype, $media) = @_;
1311
1312     return if !$media;
1313
1314     my $etype;
1315     if ($media eq 'disk') {
1316         $etype = 'images';
1317     } elsif ($media eq 'cdrom') {
1318         $etype = 'iso';
1319     } else {
1320         die "internal error";
1321     }
1322
1323     return if ($vtype eq $etype);
1324
1325     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1326 }
1327
1328 sub cleanup_drive_path {
1329     my ($opt, $storecfg, $drive) = @_;
1330
1331     # try to convert filesystem paths to volume IDs
1332
1333     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1334         ($drive->{file} !~ m|^/dev/.+|) &&
1335         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1336         ($drive->{file} !~ m/^\d+$/)) {
1337         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1338         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1339             if !$vtype;
1340         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1341         verify_media_type($opt, $vtype, $drive->{media});
1342         $drive->{file} = $volid;
1343     }
1344
1345     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1346 }
1347
1348 sub parse_hotplug_features {
1349     my ($data) = @_;
1350
1351     my $res = {};
1352
1353     return $res if $data eq '0';
1354
1355     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1356
1357     foreach my $feature (PVE::Tools::split_list($data)) {
1358         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1359             $res->{$1} = 1;
1360         } else {
1361             die "invalid hotplug feature '$feature'\n";
1362         }
1363     }
1364     return $res;
1365 }
1366
1367 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1368 sub pve_verify_hotplug_features {
1369     my ($value, $noerr) = @_;
1370
1371     return $value if parse_hotplug_features($value);
1372
1373     return if $noerr;
1374
1375     die "unable to parse hotplug option\n";
1376 }
1377
1378 sub scsi_inquiry {
1379     my($fh, $noerr) = @_;
1380
1381     my $SG_IO = 0x2285;
1382     my $SG_GET_VERSION_NUM = 0x2282;
1383
1384     my $versionbuf = "\x00" x 8;
1385     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1386     if (!$ret) {
1387         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1388         return;
1389     }
1390     my $version = unpack("I", $versionbuf);
1391     if ($version < 30000) {
1392         die "scsi generic interface too old\n"  if !$noerr;
1393         return;
1394     }
1395
1396     my $buf = "\x00" x 36;
1397     my $sensebuf = "\x00" x 8;
1398     my $cmd = pack("C x3 C x1", 0x12, 36);
1399
1400     # see /usr/include/scsi/sg.h
1401     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1402
1403     my $packet = pack(
1404         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1405     );
1406
1407     $ret = ioctl($fh, $SG_IO, $packet);
1408     if (!$ret) {
1409         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1410         return;
1411     }
1412
1413     my @res = unpack($sg_io_hdr_t, $packet);
1414     if ($res[17] || $res[18]) {
1415         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1416         return;
1417     }
1418
1419     my $res = {};
1420     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1421
1422     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1423     $res->{type} &= 0x1F;
1424
1425     return $res;
1426 }
1427
1428 sub path_is_scsi {
1429     my ($path) = @_;
1430
1431     my $fh = IO::File->new("+<$path") || return;
1432     my $res = scsi_inquiry($fh, 1);
1433     close($fh);
1434
1435     return $res;
1436 }
1437
1438 sub print_tabletdevice_full {
1439     my ($conf, $arch) = @_;
1440
1441     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1442
1443     # we use uhci for old VMs because tablet driver was buggy in older qemu
1444     my $usbbus;
1445     if ($q35 || $arch eq 'aarch64') {
1446         $usbbus = 'ehci';
1447     } else {
1448         $usbbus = 'uhci';
1449     }
1450
1451     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1452 }
1453
1454 sub print_keyboarddevice_full {
1455     my ($conf, $arch) = @_;
1456
1457     return if $arch ne 'aarch64';
1458
1459     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1460 }
1461
1462 my sub get_drive_id {
1463     my ($drive) = @_;
1464     return "$drive->{interface}$drive->{index}";
1465 }
1466
1467 sub print_drivedevice_full {
1468     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1469
1470     my $device = '';
1471     my $maxdev = 0;
1472
1473     my $drive_id = get_drive_id($drive);
1474     if ($drive->{interface} eq 'virtio') {
1475         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1476         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1477         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1478     } elsif ($drive->{interface} eq 'scsi') {
1479
1480         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1481         my $unit = $drive->{index} % $maxdev;
1482         my $devicetype = 'hd';
1483         my $path = '';
1484         if (drive_is_cdrom($drive)) {
1485             $devicetype = 'cd';
1486         } else {
1487             if ($drive->{file} =~ m|^/|) {
1488                 $path = $drive->{file};
1489                 if (my $info = path_is_scsi($path)) {
1490                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1491                         $devicetype = 'block';
1492                     } elsif ($info->{type} == 1) { # tape
1493                         $devicetype = 'generic';
1494                     }
1495                 }
1496             } else {
1497                  $path = PVE::Storage::path($storecfg, $drive->{file});
1498             }
1499
1500             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1501             my $version = extract_version($machine_type, kvm_user_version());
1502             if ($path =~ m/^iscsi\:\/\// &&
1503                !min_version($version, 4, 1)) {
1504                 $devicetype = 'generic';
1505             }
1506         }
1507
1508         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1509             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1510         } else {
1511             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1512                 .",lun=$drive->{index}";
1513         }
1514         $device .= ",drive=drive-$drive_id,id=$drive_id";
1515
1516         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1517             $device .= ",rotation_rate=1";
1518         }
1519         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1520
1521     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1522         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1523         my $controller = int($drive->{index} / $maxdev);
1524         my $unit = $drive->{index} % $maxdev;
1525
1526         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1527         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1528         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1529         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1530         # were before. Move odd ones up by 2 where they don't clash.
1531         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1532             $controller += 2 * ($unit % 2);
1533             $unit = 0;
1534         }
1535
1536         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1537
1538         $device = "ide-$devicetype";
1539         if ($drive->{interface} eq 'ide') {
1540             $device .= ",bus=ide.$controller,unit=$unit";
1541         } else {
1542             $device .= ",bus=ahci$controller.$unit";
1543         }
1544         $device .= ",drive=drive-$drive_id,id=$drive_id";
1545
1546         if ($devicetype eq 'hd') {
1547             if (my $model = $drive->{model}) {
1548                 $model = URI::Escape::uri_unescape($model);
1549                 $device .= ",model=$model";
1550             }
1551             if ($drive->{ssd}) {
1552                 $device .= ",rotation_rate=1";
1553             }
1554         }
1555         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1556     } elsif ($drive->{interface} eq 'usb') {
1557         die "implement me";
1558         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1559     } else {
1560         die "unsupported interface type";
1561     }
1562
1563     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1564
1565     if (my $serial = $drive->{serial}) {
1566         $serial = URI::Escape::uri_unescape($serial);
1567         $device .= ",serial=$serial";
1568     }
1569
1570
1571     return $device;
1572 }
1573
1574 sub get_initiator_name {
1575     my $initiator;
1576
1577     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1578     while (defined(my $line = <$fh>)) {
1579         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1580         $initiator = $1;
1581         last;
1582     }
1583     $fh->close();
1584
1585     return $initiator;
1586 }
1587
1588 my sub storage_allows_io_uring_default {
1589     my ($scfg, $cache_direct) = @_;
1590
1591     # io_uring with cache mode writeback or writethrough on krbd will hang...
1592     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1593
1594     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1595     # sometimes, just plain disable...
1596     return if $scfg && $scfg->{type} eq 'lvm';
1597
1598     # io_uring causes problems when used with CIFS since kernel 5.15
1599     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1600     return if $scfg && $scfg->{type} eq 'cifs';
1601
1602     return 1;
1603 }
1604
1605 my sub drive_uses_cache_direct {
1606     my ($drive, $scfg) = @_;
1607
1608     my $cache_direct = 0;
1609
1610     if (my $cache = $drive->{cache}) {
1611         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1612     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1613         $cache_direct = 1;
1614     }
1615
1616     return $cache_direct;
1617 }
1618
1619 sub print_drive_commandline_full {
1620     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1621
1622     my $path;
1623     my $volid = $drive->{file};
1624     my $format = $drive->{format};
1625     my $drive_id = get_drive_id($drive);
1626
1627     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1628     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1629
1630     if (drive_is_cdrom($drive)) {
1631         $path = get_iso_path($storecfg, $vmid, $volid);
1632         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1633     } else {
1634         if ($storeid) {
1635             $path = PVE::Storage::path($storecfg, $volid);
1636             $format //= qemu_img_format($scfg, $volname);
1637         } else {
1638             $path = $volid;
1639             $format //= "raw";
1640         }
1641    }
1642
1643    my $is_rbd = $path =~ m/^rbd:/;
1644
1645     my $opts = '';
1646     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1647     foreach my $o (@qemu_drive_options) {
1648         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1649     }
1650
1651     # snapshot only accepts on|off
1652     if (defined($drive->{snapshot})) {
1653         my $v = $drive->{snapshot} ? 'on' : 'off';
1654         $opts .= ",snapshot=$v";
1655     }
1656
1657     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1658         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1659     }
1660
1661     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1662         my ($dir, $qmpname) = @$type;
1663         if (my $v = $drive->{"mbps$dir"}) {
1664             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1665         }
1666         if (my $v = $drive->{"mbps${dir}_max"}) {
1667             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1668         }
1669         if (my $v = $drive->{"bps${dir}_max_length"}) {
1670             $opts .= ",throttling.bps$qmpname-max-length=$v";
1671         }
1672         if (my $v = $drive->{"iops${dir}"}) {
1673             $opts .= ",throttling.iops$qmpname=$v";
1674         }
1675         if (my $v = $drive->{"iops${dir}_max"}) {
1676             $opts .= ",throttling.iops$qmpname-max=$v";
1677         }
1678         if (my $v = $drive->{"iops${dir}_max_length"}) {
1679             $opts .= ",throttling.iops$qmpname-max-length=$v";
1680         }
1681     }
1682
1683     if ($pbs_name) {
1684         $format = "rbd" if $is_rbd;
1685         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1686             if !$format;
1687         $opts .= ",format=alloc-track,file.driver=$format";
1688     } elsif ($format) {
1689         $opts .= ",format=$format";
1690     }
1691
1692     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1693
1694     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1695
1696     if (!$drive->{aio}) {
1697         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1698             # io_uring supports all cache modes
1699             $opts .= ",aio=io_uring";
1700         } else {
1701             # aio native works only with O_DIRECT
1702             if($cache_direct) {
1703                 $opts .= ",aio=native";
1704             } else {
1705                 $opts .= ",aio=threads";
1706             }
1707         }
1708     }
1709
1710     if (!drive_is_cdrom($drive)) {
1711         my $detectzeroes;
1712         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1713             $detectzeroes = 'off';
1714         } elsif ($drive->{discard}) {
1715             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1716         } else {
1717             # This used to be our default with discard not being specified:
1718             $detectzeroes = 'on';
1719         }
1720
1721         # note: 'detect-zeroes' works per blockdev and we want it to persist
1722         # after the alloc-track is removed, so put it on 'file' directly
1723         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1724         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1725     }
1726
1727     if ($pbs_name) {
1728         $opts .= ",backing=$pbs_name";
1729         $opts .= ",auto-remove=on";
1730     }
1731
1732     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1733     my $file_param = "file";
1734     if ($pbs_name) {
1735         # non-rbd drivers require the underlying file to be a seperate block
1736         # node, so add a second .file indirection
1737         $file_param .= ".file" if !$is_rbd;
1738         $file_param .= ".filename";
1739     }
1740     my $pathinfo = $path ? "$file_param=$path," : '';
1741
1742     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1743 }
1744
1745 sub print_pbs_blockdev {
1746     my ($pbs_conf, $pbs_name) = @_;
1747     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1748     $blockdev .= ",repository=$pbs_conf->{repository}";
1749     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1750     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1751     $blockdev .= ",archive=$pbs_conf->{archive}";
1752     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1753     return $blockdev;
1754 }
1755
1756 sub print_netdevice_full {
1757     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1758
1759     my $device = $net->{model};
1760     if ($net->{model} eq 'virtio') {
1761          $device = 'virtio-net-pci';
1762      };
1763
1764     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1765     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1766     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1767         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1768         # and out of each queue plus one config interrupt and control vector queue
1769         my $vectors = $net->{queues} * 2 + 2;
1770         $tmpstr .= ",vectors=$vectors,mq=on";
1771         if (min_version($machine_version, 7, 1)) {
1772             $tmpstr .= ",packed=on";
1773         }
1774     }
1775
1776     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1777         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1778     }
1779
1780     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1781
1782     if (my $mtu = $net->{mtu}) {
1783         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1784             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1785             if ($mtu == 1) {
1786                  $mtu = $bridge_mtu;
1787             } elsif ($mtu < 576) {
1788                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1789             } elsif ($mtu > $bridge_mtu) {
1790                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1791             }
1792             $tmpstr .= ",host_mtu=$mtu";
1793         } else {
1794             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1795         }
1796     }
1797
1798     if ($use_old_bios_files) {
1799         my $romfile;
1800         if ($device eq 'virtio-net-pci') {
1801             $romfile = 'pxe-virtio.rom';
1802         } elsif ($device eq 'e1000') {
1803             $romfile = 'pxe-e1000.rom';
1804         } elsif ($device eq 'e1000e') {
1805             $romfile = 'pxe-e1000e.rom';
1806         } elsif ($device eq 'ne2k') {
1807             $romfile = 'pxe-ne2k_pci.rom';
1808         } elsif ($device eq 'pcnet') {
1809             $romfile = 'pxe-pcnet.rom';
1810         } elsif ($device eq 'rtl8139') {
1811             $romfile = 'pxe-rtl8139.rom';
1812         }
1813         $tmpstr .= ",romfile=$romfile" if $romfile;
1814     }
1815
1816     return $tmpstr;
1817 }
1818
1819 sub print_netdev_full {
1820     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1821
1822     my $i = '';
1823     if ($netid =~ m/^net(\d+)$/) {
1824         $i = int($1);
1825     }
1826
1827     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1828
1829     my $ifname = "tap${vmid}i$i";
1830
1831     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1832     die "interface name '$ifname' is too long (max 15 character)\n"
1833         if length($ifname) >= 16;
1834
1835     my $vhostparam = '';
1836     if (is_native($arch)) {
1837         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1838     }
1839
1840     my $vmname = $conf->{name} || "vm$vmid";
1841
1842     my $netdev = "";
1843     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1844
1845     if ($net->{bridge}) {
1846         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1847             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1848     } else {
1849         $netdev = "type=user,id=$netid,hostname=$vmname";
1850     }
1851
1852     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1853
1854     return $netdev;
1855 }
1856
1857 my $vga_map = {
1858     'cirrus' => 'cirrus-vga',
1859     'std' => 'VGA',
1860     'vmware' => 'vmware-svga',
1861     'virtio' => 'virtio-vga',
1862     'virtio-gl' => 'virtio-vga-gl',
1863 };
1864
1865 sub print_vga_device {
1866     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1867
1868     my $type = $vga_map->{$vga->{type}};
1869     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1870         $type = 'virtio-gpu';
1871     }
1872     my $vgamem_mb = $vga->{memory};
1873
1874     my $max_outputs = '';
1875     if ($qxlnum) {
1876         $type = $id ? 'qxl' : 'qxl-vga';
1877
1878         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1879             # set max outputs so linux can have up to 4 qxl displays with one device
1880             if (min_version($machine_version, 4, 1)) {
1881                 $max_outputs = ",max_outputs=4";
1882             }
1883         }
1884     }
1885
1886     die "no devicetype for $vga->{type}\n" if !$type;
1887
1888     my $memory = "";
1889     if ($vgamem_mb) {
1890         if ($vga->{type} =~ /^virtio/) {
1891             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1892             $memory = ",max_hostmem=$bytes";
1893         } elsif ($qxlnum) {
1894             # from https://www.spice-space.org/multiple-monitors.html
1895             $memory = ",vgamem_mb=$vga->{memory}";
1896             my $ram = $vgamem_mb * 4;
1897             my $vram = $vgamem_mb * 2;
1898             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1899         } else {
1900             $memory = ",vgamem_mb=$vga->{memory}";
1901         }
1902     } elsif ($qxlnum && $id) {
1903         $memory = ",ram_size=67108864,vram_size=33554432";
1904     }
1905
1906     my $edidoff = "";
1907     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1908         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1909     }
1910
1911     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1912     my $vgaid = "vga" . ($id // '');
1913     my $pciaddr;
1914     if ($q35 && $vgaid eq 'vga') {
1915         # the first display uses pcie.0 bus on q35 machines
1916         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1917     } else {
1918         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1919     }
1920
1921     if ($vga->{type} eq 'virtio-gl') {
1922         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1923         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1924             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1925
1926         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1927             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1928     }
1929
1930     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1931 }
1932
1933 sub parse_number_sets {
1934     my ($set) = @_;
1935     my $res = [];
1936     foreach my $part (split(/;/, $set)) {
1937         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1938             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1939             push @$res, [ $1, $2 ];
1940         } else {
1941             die "invalid range: $part\n";
1942         }
1943     }
1944     return $res;
1945 }
1946
1947 sub parse_numa {
1948     my ($data) = @_;
1949
1950     my $res = parse_property_string($numa_fmt, $data);
1951     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1952     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1953     return $res;
1954 }
1955
1956 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1957 sub parse_net {
1958     my ($data, $disable_mac_autogen) = @_;
1959
1960     my $res = eval { parse_property_string($net_fmt, $data) };
1961     if ($@) {
1962         warn $@;
1963         return;
1964     }
1965     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1966         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1967         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1968     }
1969     return $res;
1970 }
1971
1972 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1973 sub parse_ipconfig {
1974     my ($data) = @_;
1975
1976     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1977     if ($@) {
1978         warn $@;
1979         return;
1980     }
1981
1982     if ($res->{gw} && !$res->{ip}) {
1983         warn 'gateway specified without specifying an IP address';
1984         return;
1985     }
1986     if ($res->{gw6} && !$res->{ip6}) {
1987         warn 'IPv6 gateway specified without specifying an IPv6 address';
1988         return;
1989     }
1990     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1991         warn 'gateway specified together with DHCP';
1992         return;
1993     }
1994     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1995         # gw6 + auto/dhcp
1996         warn "IPv6 gateway specified together with $res->{ip6} address";
1997         return;
1998     }
1999
2000     if (!$res->{ip} && !$res->{ip6}) {
2001         return { ip => 'dhcp', ip6 => 'dhcp' };
2002     }
2003
2004     return $res;
2005 }
2006
2007 sub print_net {
2008     my $net = shift;
2009
2010     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2011 }
2012
2013 sub add_random_macs {
2014     my ($settings) = @_;
2015
2016     foreach my $opt (keys %$settings) {
2017         next if $opt !~ m/^net(\d+)$/;
2018         my $net = parse_net($settings->{$opt});
2019         next if !$net;
2020         $settings->{$opt} = print_net($net);
2021     }
2022 }
2023
2024 sub vm_is_volid_owner {
2025     my ($storecfg, $vmid, $volid) = @_;
2026
2027     if ($volid !~  m|^/|) {
2028         my ($path, $owner);
2029         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2030         if ($owner && ($owner == $vmid)) {
2031             return 1;
2032         }
2033     }
2034
2035     return;
2036 }
2037
2038 sub vmconfig_register_unused_drive {
2039     my ($storecfg, $vmid, $conf, $drive) = @_;
2040
2041     if (drive_is_cloudinit($drive)) {
2042         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2043         warn $@ if $@;
2044         delete $conf->{cloudinit};
2045     } elsif (!drive_is_cdrom($drive)) {
2046         my $volid = $drive->{file};
2047         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2048             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2049         }
2050     }
2051 }
2052
2053 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2054 my $smbios1_fmt = {
2055     uuid => {
2056         type => 'string',
2057         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2058         format_description => 'UUID',
2059         description => "Set SMBIOS1 UUID.",
2060         optional => 1,
2061     },
2062     version => {
2063         type => 'string',
2064         pattern => '[A-Za-z0-9+\/]+={0,2}',
2065         format_description => 'Base64 encoded string',
2066         description => "Set SMBIOS1 version.",
2067         optional => 1,
2068     },
2069     serial => {
2070         type => 'string',
2071         pattern => '[A-Za-z0-9+\/]+={0,2}',
2072         format_description => 'Base64 encoded string',
2073         description => "Set SMBIOS1 serial number.",
2074         optional => 1,
2075     },
2076     manufacturer => {
2077         type => 'string',
2078         pattern => '[A-Za-z0-9+\/]+={0,2}',
2079         format_description => 'Base64 encoded string',
2080         description => "Set SMBIOS1 manufacturer.",
2081         optional => 1,
2082     },
2083     product => {
2084         type => 'string',
2085         pattern => '[A-Za-z0-9+\/]+={0,2}',
2086         format_description => 'Base64 encoded string',
2087         description => "Set SMBIOS1 product ID.",
2088         optional => 1,
2089     },
2090     sku => {
2091         type => 'string',
2092         pattern => '[A-Za-z0-9+\/]+={0,2}',
2093         format_description => 'Base64 encoded string',
2094         description => "Set SMBIOS1 SKU string.",
2095         optional => 1,
2096     },
2097     family => {
2098         type => 'string',
2099         pattern => '[A-Za-z0-9+\/]+={0,2}',
2100         format_description => 'Base64 encoded string',
2101         description => "Set SMBIOS1 family string.",
2102         optional => 1,
2103     },
2104     base64 => {
2105         type => 'boolean',
2106         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2107         optional => 1,
2108     },
2109 };
2110
2111 sub parse_smbios1 {
2112     my ($data) = @_;
2113
2114     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2115     warn $@ if $@;
2116     return $res;
2117 }
2118
2119 sub print_smbios1 {
2120     my ($smbios1) = @_;
2121     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2122 }
2123
2124 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2125
2126 sub parse_watchdog {
2127     my ($value) = @_;
2128
2129     return if !$value;
2130
2131     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2132     warn $@ if $@;
2133     return $res;
2134 }
2135
2136 sub parse_guest_agent {
2137     my ($conf) = @_;
2138
2139     return {} if !defined($conf->{agent});
2140
2141     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2142     warn $@ if $@;
2143
2144     # if the agent is disabled ignore the other potentially set properties
2145     return {} if !$res->{enabled};
2146     return $res;
2147 }
2148
2149 sub get_qga_key {
2150     my ($conf, $key) = @_;
2151     return undef if !defined($conf->{agent});
2152
2153     my $agent = parse_guest_agent($conf);
2154     return $agent->{$key};
2155 }
2156
2157 sub parse_vga {
2158     my ($value) = @_;
2159
2160     return {} if !$value;
2161     my $res = eval { parse_property_string($vga_fmt, $value) };
2162     warn $@ if $@;
2163     return $res;
2164 }
2165
2166 sub parse_rng {
2167     my ($value) = @_;
2168
2169     return if !$value;
2170
2171     my $res = eval { parse_property_string($rng_fmt, $value) };
2172     warn $@ if $@;
2173     return $res;
2174 }
2175
2176 sub parse_meta_info {
2177     my ($value) = @_;
2178
2179     return if !$value;
2180
2181     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2182     warn $@ if $@;
2183     return $res;
2184 }
2185
2186 sub new_meta_info_string {
2187     my () = @_; # for now do not allow to override any value
2188
2189     return PVE::JSONSchema::print_property_string(
2190         {
2191             'creation-qemu' => kvm_user_version(),
2192             ctime => "". int(time()),
2193         },
2194         $meta_info_fmt
2195     );
2196 }
2197
2198 sub qemu_created_version_fixups {
2199     my ($conf, $forcemachine, $kvmver) = @_;
2200
2201     my $meta = parse_meta_info($conf->{meta}) // {};
2202     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2203
2204     # check if we need to apply some handling for VMs that always use the latest machine version but
2205     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2206     # would be required (we do not want to pin machine version for non-windows OS type)
2207     if (
2208         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2209         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2210         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2211         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2212     ) {
2213         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2214         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2215             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2216             # and thus with the predictable interface naming of systemd
2217             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2218         }
2219     }
2220     return;
2221 }
2222
2223 # add JSON properties for create and set function
2224 sub json_config_properties {
2225     my ($prop, $with_disk_alloc) = @_;
2226
2227     my $skip_json_config_opts = {
2228         parent => 1,
2229         snaptime => 1,
2230         vmstate => 1,
2231         runningmachine => 1,
2232         runningcpu => 1,
2233         meta => 1,
2234     };
2235
2236     foreach my $opt (keys %$confdesc) {
2237         next if $skip_json_config_opts->{$opt};
2238
2239         if ($with_disk_alloc && is_valid_drivename($opt)) {
2240             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2241         } else {
2242             $prop->{$opt} = $confdesc->{$opt};
2243         }
2244     }
2245
2246     return $prop;
2247 }
2248
2249 # Properties that we can read from an OVF file
2250 sub json_ovf_properties {
2251     my $prop = {};
2252
2253     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2254         $prop->{$device} = {
2255             type => 'string',
2256             format => 'pve-volume-id-or-absolute-path',
2257             description => "Disk image that gets imported to $device",
2258             optional => 1,
2259         };
2260     }
2261
2262     $prop->{cores} = {
2263         type => 'integer',
2264         description => "The number of CPU cores.",
2265         optional => 1,
2266     };
2267     $prop->{memory} = {
2268         type => 'integer',
2269         description => "Amount of RAM for the VM in MB.",
2270         optional => 1,
2271     };
2272     $prop->{name} = {
2273         type => 'string',
2274         description => "Name of the VM.",
2275         optional => 1,
2276     };
2277
2278     return $prop;
2279 }
2280
2281 # return copy of $confdesc_cloudinit to generate documentation
2282 sub cloudinit_config_properties {
2283
2284     return dclone($confdesc_cloudinit);
2285 }
2286
2287 sub cloudinit_pending_properties {
2288     my $p = {
2289         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2290         name => 1,
2291     };
2292     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2293     return $p;
2294 }
2295
2296 sub check_type {
2297     my ($key, $value) = @_;
2298
2299     die "unknown setting '$key'\n" if !$confdesc->{$key};
2300
2301     my $type = $confdesc->{$key}->{type};
2302
2303     if (!defined($value)) {
2304         die "got undefined value\n";
2305     }
2306
2307     if ($value =~ m/[\n\r]/) {
2308         die "property contains a line feed\n";
2309     }
2310
2311     if ($type eq 'boolean') {
2312         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2313         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2314         die "type check ('boolean') failed - got '$value'\n";
2315     } elsif ($type eq 'integer') {
2316         return int($1) if $value =~ m/^(\d+)$/;
2317         die "type check ('integer') failed - got '$value'\n";
2318     } elsif ($type eq 'number') {
2319         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2320         die "type check ('number') failed - got '$value'\n";
2321     } elsif ($type eq 'string') {
2322         if (my $fmt = $confdesc->{$key}->{format}) {
2323             PVE::JSONSchema::check_format($fmt, $value);
2324             return $value;
2325         }
2326         $value =~ s/^\"(.*)\"$/$1/;
2327         return $value;
2328     } else {
2329         die "internal error"
2330     }
2331 }
2332
2333 sub destroy_vm {
2334     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2335
2336     my $conf = PVE::QemuConfig->load_config($vmid);
2337
2338     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2339         PVE::QemuConfig->check_lock($conf);
2340     }
2341
2342     if ($conf->{template}) {
2343         # check if any base image is still used by a linked clone
2344         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2345                 my ($ds, $drive) = @_;
2346                 return if drive_is_cdrom($drive);
2347
2348                 my $volid = $drive->{file};
2349                 return if !$volid || $volid =~ m|^/|;
2350
2351                 die "base volume '$volid' is still in use by linked cloned\n"
2352                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2353
2354         });
2355     }
2356
2357     my $volids = {};
2358     my $remove_owned_drive = sub {
2359         my ($ds, $drive) = @_;
2360         return if drive_is_cdrom($drive, 1);
2361
2362         my $volid = $drive->{file};
2363         return if !$volid || $volid =~ m|^/|;
2364         return if $volids->{$volid};
2365
2366         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2367         return if !$path || !$owner || ($owner != $vmid);
2368
2369         $volids->{$volid} = 1;
2370         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2371         warn "Could not remove disk '$volid', check manually: $@" if $@;
2372     };
2373
2374     # only remove disks owned by this VM (referenced in the config)
2375     my $include_opts = {
2376         include_unused => 1,
2377         extra_keys => ['vmstate'],
2378     };
2379     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2380
2381     for my $snap (values %{$conf->{snapshots}}) {
2382         next if !defined($snap->{vmstate});
2383         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2384         next if !defined($drive);
2385         $remove_owned_drive->('vmstate', $drive);
2386     }
2387
2388     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2389
2390     if ($purge_unreferenced) { # also remove unreferenced disk
2391         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2392         PVE::Storage::foreach_volid($vmdisks, sub {
2393             my ($volid, $sid, $volname, $d) = @_;
2394             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2395             warn $@ if $@;
2396         });
2397     }
2398
2399     if (defined $replacement_conf) {
2400         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2401     } else {
2402         PVE::QemuConfig->destroy_config($vmid);
2403     }
2404 }
2405
2406 sub parse_vm_config {
2407     my ($filename, $raw, $strict) = @_;
2408
2409     return if !defined($raw);
2410
2411     my $res = {
2412         digest => Digest::SHA::sha1_hex($raw),
2413         snapshots => {},
2414         pending => {},
2415         cloudinit => {},
2416     };
2417
2418     my $handle_error = sub {
2419         my ($msg) = @_;
2420
2421         if ($strict) {
2422             die $msg;
2423         } else {
2424             warn $msg;
2425         }
2426     };
2427
2428     $filename =~ m|/qemu-server/(\d+)\.conf$|
2429         || die "got strange filename '$filename'";
2430
2431     my $vmid = $1;
2432
2433     my $conf = $res;
2434     my $descr;
2435     my $finish_description = sub {
2436         if (defined($descr)) {
2437             $descr =~ s/\s+$//;
2438             $conf->{description} = $descr;
2439         }
2440         $descr = undef;
2441     };
2442     my $section = '';
2443
2444     my @lines = split(/\n/, $raw);
2445     foreach my $line (@lines) {
2446         next if $line =~ m/^\s*$/;
2447
2448         if ($line =~ m/^\[PENDING\]\s*$/i) {
2449             $section = 'pending';
2450             $finish_description->();
2451             $conf = $res->{$section} = {};
2452             next;
2453         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2454             $section = 'cloudinit';
2455             $finish_description->();
2456             $conf = $res->{$section} = {};
2457             next;
2458
2459         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2460             $section = $1;
2461             $finish_description->();
2462             $conf = $res->{snapshots}->{$section} = {};
2463             next;
2464         }
2465
2466         if ($line =~ m/^\#(.*)$/) {
2467             $descr = '' if !defined($descr);
2468             $descr .= PVE::Tools::decode_text($1) . "\n";
2469             next;
2470         }
2471
2472         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2473             $descr = '' if !defined($descr);
2474             $descr .= PVE::Tools::decode_text($2);
2475         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2476             $conf->{snapstate} = $1;
2477         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2478             my $key = $1;
2479             my $value = $2;
2480             $conf->{$key} = $value;
2481         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2482             my $value = $1;
2483             if ($section eq 'pending') {
2484                 $conf->{delete} = $value; # we parse this later
2485             } else {
2486                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2487             }
2488         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2489             my $key = $1;
2490             my $value = $2;
2491             if ($section eq 'cloudinit') {
2492                 # ignore validation only used for informative purpose
2493                 $conf->{$key} = $value;
2494                 next;
2495             }
2496             eval { $value = check_type($key, $value); };
2497             if ($@) {
2498                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2499             } else {
2500                 $key = 'ide2' if $key eq 'cdrom';
2501                 my $fmt = $confdesc->{$key}->{format};
2502                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2503                     my $v = parse_drive($key, $value);
2504                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2505                         $v->{file} = $volid;
2506                         $value = print_drive($v);
2507                     } else {
2508                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2509                         next;
2510                     }
2511                 }
2512
2513                 $conf->{$key} = $value;
2514             }
2515         } else {
2516             $handle_error->("vm $vmid - unable to parse config: $line\n");
2517         }
2518     }
2519
2520     $finish_description->();
2521     delete $res->{snapstate}; # just to be sure
2522
2523     return $res;
2524 }
2525
2526 sub write_vm_config {
2527     my ($filename, $conf) = @_;
2528
2529     delete $conf->{snapstate}; # just to be sure
2530
2531     if ($conf->{cdrom}) {
2532         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2533         $conf->{ide2} = $conf->{cdrom};
2534         delete $conf->{cdrom};
2535     }
2536
2537     # we do not use 'smp' any longer
2538     if ($conf->{sockets}) {
2539         delete $conf->{smp};
2540     } elsif ($conf->{smp}) {
2541         $conf->{sockets} = $conf->{smp};
2542         delete $conf->{cores};
2543         delete $conf->{smp};
2544     }
2545
2546     my $used_volids = {};
2547
2548     my $cleanup_config = sub {
2549         my ($cref, $pending, $snapname) = @_;
2550
2551         foreach my $key (keys %$cref) {
2552             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2553                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2554             my $value = $cref->{$key};
2555             if ($key eq 'delete') {
2556                 die "propertry 'delete' is only allowed in [PENDING]\n"
2557                     if !$pending;
2558                 # fixme: check syntax?
2559                 next;
2560             }
2561             eval { $value = check_type($key, $value); };
2562             die "unable to parse value of '$key' - $@" if $@;
2563
2564             $cref->{$key} = $value;
2565
2566             if (!$snapname && is_valid_drivename($key)) {
2567                 my $drive = parse_drive($key, $value);
2568                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2569             }
2570         }
2571     };
2572
2573     &$cleanup_config($conf);
2574
2575     &$cleanup_config($conf->{pending}, 1);
2576
2577     foreach my $snapname (keys %{$conf->{snapshots}}) {
2578         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2579         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2580     }
2581
2582     # remove 'unusedX' settings if we re-add a volume
2583     foreach my $key (keys %$conf) {
2584         my $value = $conf->{$key};
2585         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2586             delete $conf->{$key};
2587         }
2588     }
2589
2590     my $generate_raw_config = sub {
2591         my ($conf, $pending) = @_;
2592
2593         my $raw = '';
2594
2595         # add description as comment to top of file
2596         if (defined(my $descr = $conf->{description})) {
2597             if ($descr) {
2598                 foreach my $cl (split(/\n/, $descr)) {
2599                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2600                 }
2601             } else {
2602                 $raw .= "#\n" if $pending;
2603             }
2604         }
2605
2606         foreach my $key (sort keys %$conf) {
2607             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2608             $raw .= "$key: $conf->{$key}\n";
2609         }
2610         return $raw;
2611     };
2612
2613     my $raw = &$generate_raw_config($conf);
2614
2615     if (scalar(keys %{$conf->{pending}})){
2616         $raw .= "\n[PENDING]\n";
2617         $raw .= &$generate_raw_config($conf->{pending}, 1);
2618     }
2619
2620     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2621         $raw .= "\n[special:cloudinit]\n";
2622         $raw .= &$generate_raw_config($conf->{cloudinit});
2623     }
2624
2625     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2626         $raw .= "\n[$snapname]\n";
2627         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2628     }
2629
2630     return $raw;
2631 }
2632
2633 sub load_defaults {
2634
2635     my $res = {};
2636
2637     # we use static defaults from our JSON schema configuration
2638     foreach my $key (keys %$confdesc) {
2639         if (defined(my $default = $confdesc->{$key}->{default})) {
2640             $res->{$key} = $default;
2641         }
2642     }
2643
2644     return $res;
2645 }
2646
2647 sub config_list {
2648     my $vmlist = PVE::Cluster::get_vmlist();
2649     my $res = {};
2650     return $res if !$vmlist || !$vmlist->{ids};
2651     my $ids = $vmlist->{ids};
2652     my $nodename = nodename();
2653
2654     foreach my $vmid (keys %$ids) {
2655         my $d = $ids->{$vmid};
2656         next if !$d->{node} || $d->{node} ne $nodename;
2657         next if !$d->{type} || $d->{type} ne 'qemu';
2658         $res->{$vmid}->{exists} = 1;
2659     }
2660     return $res;
2661 }
2662
2663 # test if VM uses local resources (to prevent migration)
2664 sub check_local_resources {
2665     my ($conf, $noerr) = @_;
2666
2667     my @loc_res = ();
2668     my $mapped_res = [];
2669
2670     my $nodelist = PVE::Cluster::get_nodelist();
2671     my $pci_map = PVE::Mapping::PCI::config();
2672     my $usb_map = PVE::Mapping::USB::config();
2673
2674     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2675
2676     my $add_missing_mapping = sub {
2677         my ($type, $key, $id) = @_;
2678         for my $node (@$nodelist) {
2679             my $entry;
2680             if ($type eq 'pci') {
2681                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2682             } elsif ($type eq 'usb') {
2683                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2684             }
2685             if (!scalar($entry->@*)) {
2686                 push @{$missing_mappings_by_node->{$node}}, $key;
2687             }
2688         }
2689     };
2690
2691     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2692     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2693
2694     push @loc_res, "ivshmem" if $conf->{ivshmem};
2695
2696     foreach my $k (keys %$conf) {
2697         if ($k =~ m/^usb/) {
2698             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2699             next if $entry->{host} =~ m/^spice$/i;
2700             if ($entry->{mapping}) {
2701                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2702                 push @$mapped_res, $k;
2703             }
2704         }
2705         if ($k =~ m/^hostpci/) {
2706             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2707             if ($entry->{mapping}) {
2708                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2709                 push @$mapped_res, $k;
2710             }
2711         }
2712         # sockets are safe: they will recreated be on the target side post-migrate
2713         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2714         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2715     }
2716
2717     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2718
2719     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2720 }
2721
2722 # check if used storages are available on all nodes (use by migrate)
2723 sub check_storage_availability {
2724     my ($storecfg, $conf, $node) = @_;
2725
2726     PVE::QemuConfig->foreach_volume($conf, sub {
2727         my ($ds, $drive) = @_;
2728
2729         my $volid = $drive->{file};
2730         return if !$volid;
2731
2732         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2733         return if !$sid;
2734
2735         # check if storage is available on both nodes
2736         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2737         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2738
2739         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2740
2741         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2742             if !$scfg->{content}->{$vtype};
2743    });
2744 }
2745
2746 # list nodes where all VM images are available (used by has_feature API)
2747 sub shared_nodes {
2748     my ($conf, $storecfg) = @_;
2749
2750     my $nodelist = PVE::Cluster::get_nodelist();
2751     my $nodehash = { map { $_ => 1 } @$nodelist };
2752     my $nodename = nodename();
2753
2754     PVE::QemuConfig->foreach_volume($conf, sub {
2755         my ($ds, $drive) = @_;
2756
2757         my $volid = $drive->{file};
2758         return if !$volid;
2759
2760         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2761         if ($storeid) {
2762             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2763             if ($scfg->{disable}) {
2764                 $nodehash = {};
2765             } elsif (my $avail = $scfg->{nodes}) {
2766                 foreach my $node (keys %$nodehash) {
2767                     delete $nodehash->{$node} if !$avail->{$node};
2768                 }
2769             } elsif (!$scfg->{shared}) {
2770                 foreach my $node (keys %$nodehash) {
2771                     delete $nodehash->{$node} if $node ne $nodename
2772                 }
2773             }
2774         }
2775     });
2776
2777     return $nodehash
2778 }
2779
2780 sub check_local_storage_availability {
2781     my ($conf, $storecfg) = @_;
2782
2783     my $nodelist = PVE::Cluster::get_nodelist();
2784     my $nodehash = { map { $_ => {} } @$nodelist };
2785
2786     PVE::QemuConfig->foreach_volume($conf, sub {
2787         my ($ds, $drive) = @_;
2788
2789         my $volid = $drive->{file};
2790         return if !$volid;
2791
2792         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2793         if ($storeid) {
2794             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2795
2796             if ($scfg->{disable}) {
2797                 foreach my $node (keys %$nodehash) {
2798                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2799                 }
2800             } elsif (my $avail = $scfg->{nodes}) {
2801                 foreach my $node (keys %$nodehash) {
2802                     if (!$avail->{$node}) {
2803                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2804                     }
2805                 }
2806             }
2807         }
2808     });
2809
2810     foreach my $node (values %$nodehash) {
2811         if (my $unavail = $node->{unavailable_storages}) {
2812             $node->{unavailable_storages} = [ sort keys %$unavail ];
2813         }
2814     }
2815
2816     return $nodehash
2817 }
2818
2819 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2820 sub check_running {
2821     my ($vmid, $nocheck, $node) = @_;
2822
2823     # $nocheck is set when called during a migration, in which case the config
2824     # file might still or already reside on the *other* node
2825     # - because rename has already happened, and current node is source
2826     # - because rename hasn't happened yet, and current node is target
2827     # - because rename has happened, current node is target, but hasn't yet
2828     # processed it yet
2829     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2830     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2831 }
2832
2833 sub vzlist {
2834
2835     my $vzlist = config_list();
2836
2837     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2838
2839     while (defined(my $de = $fd->read)) {
2840         next if $de !~ m/^(\d+)\.pid$/;
2841         my $vmid = $1;
2842         next if !defined($vzlist->{$vmid});
2843         if (my $pid = check_running($vmid)) {
2844             $vzlist->{$vmid}->{pid} = $pid;
2845         }
2846     }
2847
2848     return $vzlist;
2849 }
2850
2851 our $vmstatus_return_properties = {
2852     vmid => get_standard_option('pve-vmid'),
2853     status => {
2854         description => "QEMU process status.",
2855         type => 'string',
2856         enum => ['stopped', 'running'],
2857     },
2858     maxmem => {
2859         description => "Maximum memory in bytes.",
2860         type => 'integer',
2861         optional => 1,
2862         renderer => 'bytes',
2863     },
2864     maxdisk => {
2865         description => "Root disk size in bytes.",
2866         type => 'integer',
2867         optional => 1,
2868         renderer => 'bytes',
2869     },
2870     name => {
2871         description => "VM name.",
2872         type => 'string',
2873         optional => 1,
2874     },
2875     qmpstatus => {
2876         description => "VM run state from the 'query-status' QMP monitor command.",
2877         type => 'string',
2878         optional => 1,
2879     },
2880     pid => {
2881         description => "PID of running qemu process.",
2882         type => 'integer',
2883         optional => 1,
2884     },
2885     uptime => {
2886         description => "Uptime.",
2887         type => 'integer',
2888         optional => 1,
2889         renderer => 'duration',
2890     },
2891     cpus => {
2892         description => "Maximum usable CPUs.",
2893         type => 'number',
2894         optional => 1,
2895     },
2896     lock => {
2897         description => "The current config lock, if any.",
2898         type => 'string',
2899         optional => 1,
2900     },
2901     tags => {
2902         description  => "The current configured tags, if any",
2903         type => 'string',
2904         optional => 1,
2905     },
2906     'running-machine' => {
2907         description => "The currently running machine type (if running).",
2908         type => 'string',
2909         optional => 1,
2910     },
2911     'running-qemu' => {
2912         description => "The currently running QEMU version (if running).",
2913         type => 'string',
2914         optional => 1,
2915     },
2916 };
2917
2918 my $last_proc_pid_stat;
2919
2920 # get VM status information
2921 # This must be fast and should not block ($full == false)
2922 # We only query KVM using QMP if $full == true (this can be slow)
2923 sub vmstatus {
2924     my ($opt_vmid, $full) = @_;
2925
2926     my $res = {};
2927
2928     my $storecfg = PVE::Storage::config();
2929
2930     my $list = vzlist();
2931     my $defaults = load_defaults();
2932
2933     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2934
2935     my $cpucount = $cpuinfo->{cpus} || 1;
2936
2937     foreach my $vmid (keys %$list) {
2938         next if $opt_vmid && ($vmid ne $opt_vmid);
2939
2940         my $conf = PVE::QemuConfig->load_config($vmid);
2941
2942         my $d = { vmid => int($vmid) };
2943         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2944
2945         # fixme: better status?
2946         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2947
2948         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2949         if (defined($size)) {
2950             $d->{disk} = 0; # no info available
2951             $d->{maxdisk} = $size;
2952         } else {
2953             $d->{disk} = 0;
2954             $d->{maxdisk} = 0;
2955         }
2956
2957         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2958             * ($conf->{cores} || $defaults->{cores});
2959         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2960         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2961
2962         $d->{name} = $conf->{name} || "VM $vmid";
2963         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2964             : $defaults->{memory}*(1024*1024);
2965
2966         if ($conf->{balloon}) {
2967             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2968             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2969                 : $defaults->{shares};
2970         }
2971
2972         $d->{uptime} = 0;
2973         $d->{cpu} = 0;
2974         $d->{mem} = 0;
2975
2976         $d->{netout} = 0;
2977         $d->{netin} = 0;
2978
2979         $d->{diskread} = 0;
2980         $d->{diskwrite} = 0;
2981
2982         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2983
2984         $d->{serial} = 1 if conf_has_serial($conf);
2985         $d->{lock} = $conf->{lock} if $conf->{lock};
2986         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2987
2988         $res->{$vmid} = $d;
2989     }
2990
2991     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2992     foreach my $dev (keys %$netdev) {
2993         next if $dev !~ m/^tap([1-9]\d*)i/;
2994         my $vmid = $1;
2995         my $d = $res->{$vmid};
2996         next if !$d;
2997
2998         $d->{netout} += $netdev->{$dev}->{receive};
2999         $d->{netin} += $netdev->{$dev}->{transmit};
3000
3001         if ($full) {
3002             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
3003             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
3004         }
3005
3006     }
3007
3008     my $ctime = gettimeofday;
3009
3010     foreach my $vmid (keys %$list) {
3011
3012         my $d = $res->{$vmid};
3013         my $pid = $d->{pid};
3014         next if !$pid;
3015
3016         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3017         next if !$pstat; # not running
3018
3019         my $used = $pstat->{utime} + $pstat->{stime};
3020
3021         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3022
3023         if ($pstat->{vsize}) {
3024             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3025         }
3026
3027         my $old = $last_proc_pid_stat->{$pid};
3028         if (!$old) {
3029             $last_proc_pid_stat->{$pid} = {
3030                 time => $ctime,
3031                 used => $used,
3032                 cpu => 0,
3033             };
3034             next;
3035         }
3036
3037         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3038
3039         if ($dtime > 1000) {
3040             my $dutime = $used -  $old->{used};
3041
3042             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3043             $last_proc_pid_stat->{$pid} = {
3044                 time => $ctime,
3045                 used => $used,
3046                 cpu => $d->{cpu},
3047             };
3048         } else {
3049             $d->{cpu} = $old->{cpu};
3050         }
3051     }
3052
3053     return $res if !$full;
3054
3055     my $qmpclient = PVE::QMPClient->new();
3056
3057     my $ballooncb = sub {
3058         my ($vmid, $resp) = @_;
3059
3060         my $info = $resp->{'return'};
3061         return if !$info->{max_mem};
3062
3063         my $d = $res->{$vmid};
3064
3065         # use memory assigned to VM
3066         $d->{maxmem} = $info->{max_mem};
3067         $d->{balloon} = $info->{actual};
3068
3069         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3070             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3071             $d->{freemem} = $info->{free_mem};
3072         }
3073
3074         $d->{ballooninfo} = $info;
3075     };
3076
3077     my $blockstatscb = sub {
3078         my ($vmid, $resp) = @_;
3079         my $data = $resp->{'return'} || [];
3080         my $totalrdbytes = 0;
3081         my $totalwrbytes = 0;
3082
3083         for my $blockstat (@$data) {
3084             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3085             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3086
3087             $blockstat->{device} =~ s/drive-//;
3088             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3089         }
3090         $res->{$vmid}->{diskread} = $totalrdbytes;
3091         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3092     };
3093
3094     my $machinecb = sub {