PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 123     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 124     type => 'string',
 125     format => 'storage-pair-list',
 126     optional => 1,
 127 });
 128
 129 #no warnings 'redefine';
 130
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0
 167     },
 168     type => {
 169         description => "Select the agent type",
 170         type => 'string',
 171         default => 'virtio',
 172         optional => 1,
 173         enum => [qw(virtio isa)],
 174     },
 175 };
 176
 177 my $vga_fmt = {
 178     type => {
 179         description => "Select the VGA type.",
 180         type => 'string',
 181         default => 'std',
 182         optional => 1,
 183         default_key => 1,
 184         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 185     },
 186     memory => {
 187         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 188         type => 'integer',
 189         optional => 1,
 190         minimum => 4,
 191         maximum => 512,
 192     },
 193 };
 194
 195 my $ivshmem_fmt = {
 196     size => {
 197         type => 'integer',
 198         minimum => 1,
 199         description => "The size of the file in MB.",
 200     },
 201     name => {
 202         type => 'string',
 203         pattern => '[a-zA-Z0-9\-]+',
 204         optional => 1,
 205         format_description => 'string',
 206         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 207     },
 208 };
 209
 210 my $audio_fmt = {
 211     device => {
 212         type => 'string',
 213         enum => [qw(ich9-intel-hda intel-hda AC97)],
 214         description =>  "Configure an audio device."
 215     },
 216     driver =>  {
 217         type => 'string',
 218         enum => ['spice', 'none'],
 219         default => 'spice',
 220         optional => 1,
 221         description => "Driver backend for the audio device."
 222     },
 223 };
 224
 225 my $spice_enhancements_fmt = {
 226     foldersharing => {
 227         type => 'boolean',
 228         optional => 1,
 229         default => '0',
 230         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 231     },
 232     videostreaming =>  {
 233         type => 'string',
 234         enum => ['off', 'all', 'filter'],
 235         default => 'off',
 236         optional => 1,
 237         description => "Enable video streaming. Uses compression for detected video streams."
 238     },
 239 };
 240
 241 my $rng_fmt = {
 242     source => {
 243         type => 'string',
 244         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 245         default_key => 1,
 246         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 247             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 248             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 249             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 250             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 251             ." a hardware RNG from the host.",
 252     },
 253     max_bytes => {
 254         type => 'integer',
 255         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 256             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 257             ." `0` to disable limiting (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 261         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 262         # reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 268             ." the guest to retrieve another 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $meta_info_fmt = {
 275     'ctime' => {
 276         type => 'integer',
 277         description => "The guest creation timestamp as UNIX epoch time",
 278         minimum => 0,
 279         optional => 1,
 280     },
 281     'creation-qemu' => {
 282         type => 'string',
 283         description => "The QEMU (machine) version from the time this VM was created.",
 284         pattern => '\d+(\.\d+)+',
 285         optional => 1,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 1,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. The default is read from the"
 371             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710     meta => {
 711         type => 'string',
 712         format => $meta_info_fmt,
 713         description => "Some (read-only) meta-information about this guest.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.",
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.",
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037     my ($volid, $noerr) = @_;
1038
1039     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1040         return $volid;
1041     }
1042
1043     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1044     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1045     if ($@) {
1046         return if $noerr;
1047         die $@;
1048     }
1049     return $volid;
1050 }
1051
1052 my $usb_fmt = {
1053     host => {
1054         default_key => 1,
1055         type => 'string', format => 'pve-qm-usb-device',
1056         format_description => 'HOSTUSBDEVICE|spice',
1057         description => <<EODESCR,
1058 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1059
1060  'bus-port(.port)*' (decimal numbers) or
1061  'vendor_id:product_id' (hexadeciaml numbers) or
1062  'spice'
1063
1064 You can use the 'lsusb -t' command to list existing usb devices.
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 The value 'spice' can be used to add a usb redirection devices for spice.
1070 EODESCR
1071     },
1072     usb3 => {
1073         optional => 1,
1074         type => 'boolean',
1075         description => "Specifies whether if given host option is a USB3 device or port.",
1076         default => 0,
1077     },
1078 };
1079
1080 my $usbdesc = {
1081     optional => 1,
1082     type => 'string', format => $usb_fmt,
1083     description => "Configure an USB device (n is 0 to 4).",
1084 };
1085 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1086
1087 my $serialdesc = {
1088         optional => 1,
1089         type => 'string',
1090         pattern => '(/dev/.+|socket)',
1091         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1092         verbose_description =>  <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1096
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1098 use with special care.
1099
1100 CAUTION: Experimental! User reported problems with this option.
1101 EODESCR
1102 };
1103
1104 my $paralleldesc= {
1105         optional => 1,
1106         type => 'string',
1107         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1108         description =>  "Map host parallel devices (n is 0 to 2).",
1109         verbose_description =>  <<EODESCR,
1110 Map host parallel devices (n is 0 to 2).
1111
1112 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1113 machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1129 }
1130
1131 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1132     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1133 }
1134
1135 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1136     $confdesc->{"usb$i"} = $usbdesc;
1137 }
1138
1139 my $boot_fmt = {
1140     legacy => {
1141         optional => 1,
1142         default_key => 1,
1143         type => 'string',
1144         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1145                      . " Deprecated, use 'order=' instead.",
1146         pattern => '[acdn]{1,4}',
1147         format_description => "[acdn]{1,4}",
1148
1149         # note: this is also the fallback if boot: is not given at all
1150         default => 'cdn',
1151     },
1152     order => {
1153         optional => 1,
1154         type => 'string',
1155         format => 'pve-qm-bootdev-list',
1156         format_description => "device[;device...]",
1157         description => <<EODESC,
1158 The guest will attempt to boot from devices in the order they appear here.
1159
1160 Disks, optical drives and passed-through storage USB devices will be directly
1161 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1162 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1163
1164 Note that only devices in this list will be marked as bootable and thus loaded
1165 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1166 (e.g. software-raid), you need to specify all of them here.
1167
1168 Overrides the deprecated 'legacy=[acdn]*' value when given.
1169 EODESC
1170     },
1171 };
1172 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1173
1174 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1175 sub verify_bootdev {
1176     my ($dev, $noerr) = @_;
1177
1178     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1179     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1180
1181     my $check = sub {
1182         my ($base) = @_;
1183         return 0 if $dev !~ m/^$base\d+$/;
1184         return 0 if !$confdesc->{$dev};
1185         return 1;
1186     };
1187
1188     return $dev if $check->("net");
1189     return $dev if $check->("usb");
1190     return $dev if $check->("hostpci");
1191
1192     return if $noerr;
1193     die "invalid boot device '$dev'\n";
1194 }
1195
1196 sub print_bootorder {
1197     my ($devs) = @_;
1198     return "" if !@$devs;
1199     my $data = { order => join(';', @$devs) };
1200     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1201 }
1202
1203 my $kvm_api_version = 0;
1204
1205 sub kvm_version {
1206     return $kvm_api_version if $kvm_api_version;
1207
1208     open my $fh, '<', '/dev/kvm' or return;
1209
1210     # 0xae00 => KVM_GET_API_VERSION
1211     $kvm_api_version = ioctl($fh, 0xae00, 0);
1212     close($fh);
1213
1214     return $kvm_api_version;
1215 }
1216
1217 my $kvm_user_version = {};
1218 my $kvm_mtime = {};
1219
1220 sub kvm_user_version {
1221     my ($binary) = @_;
1222
1223     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1224     my $st = stat($binary);
1225
1226     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1227     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1228         $cachedmtime == $st->mtime;
1229
1230     $kvm_user_version->{$binary} = 'unknown';
1231     $kvm_mtime->{$binary} = $st->mtime;
1232
1233     my $code = sub {
1234         my $line = shift;
1235         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1236             $kvm_user_version->{$binary} = $2;
1237         }
1238     };
1239
1240     eval { run_command([$binary, '--version'], outfunc => $code); };
1241     warn $@ if $@;
1242
1243     return $kvm_user_version->{$binary};
1244
1245 }
1246 my sub extract_version {
1247     my ($machine_type, $version) = @_;
1248     $version = kvm_user_version() if !defined($version);
1249     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1250 }
1251
1252 sub kernel_has_vhost_net {
1253     return -c '/dev/vhost-net';
1254 }
1255
1256 sub option_exists {
1257     my $key = shift;
1258     return defined($confdesc->{$key});
1259 }
1260
1261 my $cdrom_path;
1262 sub get_cdrom_path {
1263
1264     return  $cdrom_path if $cdrom_path;
1265
1266     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1267     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1268     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1269 }
1270
1271 sub get_iso_path {
1272     my ($storecfg, $vmid, $cdrom) = @_;
1273
1274     if ($cdrom eq 'cdrom') {
1275         return get_cdrom_path();
1276     } elsif ($cdrom eq 'none') {
1277         return '';
1278     } elsif ($cdrom =~ m|^/|) {
1279         return $cdrom;
1280     } else {
1281         return PVE::Storage::path($storecfg, $cdrom);
1282     }
1283 }
1284
1285 # try to convert old style file names to volume IDs
1286 sub filename_to_volume_id {
1287     my ($vmid, $file, $media) = @_;
1288
1289      if (!($file eq 'none' || $file eq 'cdrom' ||
1290           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1291
1292         return if $file =~ m|/|;
1293
1294         if ($media && $media eq 'cdrom') {
1295             $file = "local:iso/$file";
1296         } else {
1297             $file = "local:$vmid/$file";
1298         }
1299     }
1300
1301     return $file;
1302 }
1303
1304 sub verify_media_type {
1305     my ($opt, $vtype, $media) = @_;
1306
1307     return if !$media;
1308
1309     my $etype;
1310     if ($media eq 'disk') {
1311         $etype = 'images';
1312     } elsif ($media eq 'cdrom') {
1313         $etype = 'iso';
1314     } else {
1315         die "internal error";
1316     }
1317
1318     return if ($vtype eq $etype);
1319
1320     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1321 }
1322
1323 sub cleanup_drive_path {
1324     my ($opt, $storecfg, $drive) = @_;
1325
1326     # try to convert filesystem paths to volume IDs
1327
1328     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1329         ($drive->{file} !~ m|^/dev/.+|) &&
1330         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1331         ($drive->{file} !~ m/^\d+$/)) {
1332         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1333         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1334             if !$vtype;
1335         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1336         verify_media_type($opt, $vtype, $drive->{media});
1337         $drive->{file} = $volid;
1338     }
1339
1340     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1341 }
1342
1343 sub parse_hotplug_features {
1344     my ($data) = @_;
1345
1346     my $res = {};
1347
1348     return $res if $data eq '0';
1349
1350     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1351
1352     foreach my $feature (PVE::Tools::split_list($data)) {
1353         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1354             $res->{$1} = 1;
1355         } else {
1356             die "invalid hotplug feature '$feature'\n";
1357         }
1358     }
1359     return $res;
1360 }
1361
1362 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1363 sub pve_verify_hotplug_features {
1364     my ($value, $noerr) = @_;
1365
1366     return $value if parse_hotplug_features($value);
1367
1368     return if $noerr;
1369
1370     die "unable to parse hotplug option\n";
1371 }
1372
1373 sub scsi_inquiry {
1374     my($fh, $noerr) = @_;
1375
1376     my $SG_IO = 0x2285;
1377     my $SG_GET_VERSION_NUM = 0x2282;
1378
1379     my $versionbuf = "\x00" x 8;
1380     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1381     if (!$ret) {
1382         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1383         return;
1384     }
1385     my $version = unpack("I", $versionbuf);
1386     if ($version < 30000) {
1387         die "scsi generic interface too old\n"  if !$noerr;
1388         return;
1389     }
1390
1391     my $buf = "\x00" x 36;
1392     my $sensebuf = "\x00" x 8;
1393     my $cmd = pack("C x3 C x1", 0x12, 36);
1394
1395     # see /usr/include/scsi/sg.h
1396     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1397
1398     my $packet = pack(
1399         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1400     );
1401
1402     $ret = ioctl($fh, $SG_IO, $packet);
1403     if (!$ret) {
1404         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1405         return;
1406     }
1407
1408     my @res = unpack($sg_io_hdr_t, $packet);
1409     if ($res[17] || $res[18]) {
1410         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my $res = {};
1415     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1416
1417     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1418     $res->{type} &= 0x1F;
1419
1420     return $res;
1421 }
1422
1423 sub path_is_scsi {
1424     my ($path) = @_;
1425
1426     my $fh = IO::File->new("+<$path") || return;
1427     my $res = scsi_inquiry($fh, 1);
1428     close($fh);
1429
1430     return $res;
1431 }
1432
1433 sub print_tabletdevice_full {
1434     my ($conf, $arch) = @_;
1435
1436     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1437
1438     # we use uhci for old VMs because tablet driver was buggy in older qemu
1439     my $usbbus;
1440     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1441         $usbbus = 'ehci';
1442     } else {
1443         $usbbus = 'uhci';
1444     }
1445
1446     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1447 }
1448
1449 sub print_keyboarddevice_full {
1450     my ($conf, $arch) = @_;
1451
1452     return if $arch ne 'aarch64';
1453
1454     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1455 }
1456
1457 my sub get_drive_id {
1458     my ($drive) = @_;
1459     return "$drive->{interface}$drive->{index}";
1460 }
1461
1462 sub print_drivedevice_full {
1463     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1464
1465     my $device = '';
1466     my $maxdev = 0;
1467
1468     my $drive_id = get_drive_id($drive);
1469     if ($drive->{interface} eq 'virtio') {
1470         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1471         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1472         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1473     } elsif ($drive->{interface} eq 'scsi') {
1474
1475         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1476         my $unit = $drive->{index} % $maxdev;
1477         my $devicetype = 'hd';
1478         my $path = '';
1479         if (drive_is_cdrom($drive)) {
1480             $devicetype = 'cd';
1481         } else {
1482             if ($drive->{file} =~ m|^/|) {
1483                 $path = $drive->{file};
1484                 if (my $info = path_is_scsi($path)) {
1485                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1486                         $devicetype = 'block';
1487                     } elsif ($info->{type} == 1) { # tape
1488                         $devicetype = 'generic';
1489                     }
1490                 }
1491             } else {
1492                  $path = PVE::Storage::path($storecfg, $drive->{file});
1493             }
1494
1495             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1496             my $version = extract_version($machine_type, kvm_user_version());
1497             if ($path =~ m/^iscsi\:\/\// &&
1498                !min_version($version, 4, 1)) {
1499                 $devicetype = 'generic';
1500             }
1501         }
1502
1503         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1504             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1505         } else {
1506             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1507                 .",lun=$drive->{index}";
1508         }
1509         $device .= ",drive=drive-$drive_id,id=$drive_id";
1510
1511         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1512             $device .= ",rotation_rate=1";
1513         }
1514         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1515
1516     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1517         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1518         my $controller = int($drive->{index} / $maxdev);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1521
1522         $device = "ide-$devicetype";
1523         if ($drive->{interface} eq 'ide') {
1524             $device .= ",bus=ide.$controller,unit=$unit";
1525         } else {
1526             $device .= ",bus=ahci$controller.$unit";
1527         }
1528         $device .= ",drive=drive-$drive_id,id=$drive_id";
1529
1530         if ($devicetype eq 'hd') {
1531             if (my $model = $drive->{model}) {
1532                 $model = URI::Escape::uri_unescape($model);
1533                 $device .= ",model=$model";
1534             }
1535             if ($drive->{ssd}) {
1536                 $device .= ",rotation_rate=1";
1537             }
1538         }
1539         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1540     } elsif ($drive->{interface} eq 'usb') {
1541         die "implement me";
1542         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1543     } else {
1544         die "unsupported interface type";
1545     }
1546
1547     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1548
1549     if (my $serial = $drive->{serial}) {
1550         $serial = URI::Escape::uri_unescape($serial);
1551         $device .= ",serial=$serial";
1552     }
1553
1554
1555     return $device;
1556 }
1557
1558 sub get_initiator_name {
1559     my $initiator;
1560
1561     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1562     while (defined(my $line = <$fh>)) {
1563         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1564         $initiator = $1;
1565         last;
1566     }
1567     $fh->close();
1568
1569     return $initiator;
1570 }
1571
1572 sub print_drive_commandline_full {
1573     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1574
1575     my $path;
1576     my $volid = $drive->{file};
1577     my $format = $drive->{format};
1578     my $drive_id = get_drive_id($drive);
1579
1580     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1581     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1582
1583     if (drive_is_cdrom($drive)) {
1584         $path = get_iso_path($storecfg, $vmid, $volid);
1585         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1586     } else {
1587         if ($storeid) {
1588             $path = PVE::Storage::path($storecfg, $volid);
1589             $format //= qemu_img_format($scfg, $volname);
1590         } else {
1591             $path = $volid;
1592             $format //= "raw";
1593         }
1594    }
1595
1596    my $is_rbd = $path =~ m/^rbd:/;
1597
1598     my $opts = '';
1599     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1600     foreach my $o (@qemu_drive_options) {
1601         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1602     }
1603
1604     # snapshot only accepts on|off
1605     if (defined($drive->{snapshot})) {
1606         my $v = $drive->{snapshot} ? 'on' : 'off';
1607         $opts .= ",snapshot=$v";
1608     }
1609
1610     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1611         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1612     }
1613
1614     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1615         my ($dir, $qmpname) = @$type;
1616         if (my $v = $drive->{"mbps$dir"}) {
1617             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1618         }
1619         if (my $v = $drive->{"mbps${dir}_max"}) {
1620             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1621         }
1622         if (my $v = $drive->{"bps${dir}_max_length"}) {
1623             $opts .= ",throttling.bps$qmpname-max-length=$v";
1624         }
1625         if (my $v = $drive->{"iops${dir}"}) {
1626             $opts .= ",throttling.iops$qmpname=$v";
1627         }
1628         if (my $v = $drive->{"iops${dir}_max"}) {
1629             $opts .= ",throttling.iops$qmpname-max=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}_max_length"}) {
1632             $opts .= ",throttling.iops$qmpname-max-length=$v";
1633         }
1634     }
1635
1636     if ($pbs_name) {
1637         $format = "rbd" if $is_rbd;
1638         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1639             if !$format;
1640         $opts .= ",format=alloc-track,file.driver=$format";
1641     } elsif ($format) {
1642         $opts .= ",format=$format";
1643     }
1644
1645     my $cache_direct = 0;
1646
1647     if (my $cache = $drive->{cache}) {
1648         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1649     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1650         $opts .= ",cache=none";
1651         $cache_direct = 1;
1652     }
1653
1654     # io_uring with cache mode writeback or writethrough on krbd will hang...
1655     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1656
1657     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1658     # sometimes, just plain disable...
1659     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1660
1661     if (!$drive->{aio}) {
1662         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1663             # io_uring supports all cache modes
1664             $opts .= ",aio=io_uring";
1665         } else {
1666             # aio native works only with O_DIRECT
1667             if($cache_direct) {
1668                 $opts .= ",aio=native";
1669             } else {
1670                 $opts .= ",aio=threads";
1671             }
1672         }
1673     }
1674
1675     if (!drive_is_cdrom($drive)) {
1676         my $detectzeroes;
1677         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1678             $detectzeroes = 'off';
1679         } elsif ($drive->{discard}) {
1680             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1681         } else {
1682             # This used to be our default with discard not being specified:
1683             $detectzeroes = 'on';
1684         }
1685
1686         # note: 'detect-zeroes' works per blockdev and we want it to persist
1687         # after the alloc-track is removed, so put it on 'file' directly
1688         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1689         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1690     }
1691
1692     if ($pbs_name) {
1693         $opts .= ",backing=$pbs_name";
1694         $opts .= ",auto-remove=on";
1695     }
1696
1697     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1698     my $file_param = "file";
1699     if ($pbs_name) {
1700         # non-rbd drivers require the underlying file to be a seperate block
1701         # node, so add a second .file indirection
1702         $file_param .= ".file" if !$is_rbd;
1703         $file_param .= ".filename";
1704     }
1705     my $pathinfo = $path ? "$file_param=$path," : '';
1706
1707     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1708 }
1709
1710 sub print_pbs_blockdev {
1711     my ($pbs_conf, $pbs_name) = @_;
1712     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1713     $blockdev .= ",repository=$pbs_conf->{repository}";
1714     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1715     $blockdev .= ",archive=$pbs_conf->{archive}";
1716     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1717     return $blockdev;
1718 }
1719
1720 sub print_netdevice_full {
1721     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1722
1723     my $device = $net->{model};
1724     if ($net->{model} eq 'virtio') {
1725          $device = 'virtio-net-pci';
1726      };
1727
1728     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1729     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1730     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1731         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1732         # and out of each queue plus one config interrupt and control vector queue
1733         my $vectors = $net->{queues} * 2 + 2;
1734         $tmpstr .= ",vectors=$vectors,mq=on";
1735     }
1736     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1737
1738     if (my $mtu = $net->{mtu}) {
1739         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1740             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1741             if ($mtu == 1) {
1742                  $mtu = $bridge_mtu;
1743             } elsif ($mtu < 576) {
1744                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1745             } elsif ($mtu > $bridge_mtu) {
1746                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1747             }
1748             $tmpstr .= ",host_mtu=$mtu";
1749         } else {
1750             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1751         }
1752     }
1753
1754     if ($use_old_bios_files) {
1755         my $romfile;
1756         if ($device eq 'virtio-net-pci') {
1757             $romfile = 'pxe-virtio.rom';
1758         } elsif ($device eq 'e1000') {
1759             $romfile = 'pxe-e1000.rom';
1760         } elsif ($device eq 'e1000e') {
1761             $romfile = 'pxe-e1000e.rom';
1762         } elsif ($device eq 'ne2k') {
1763             $romfile = 'pxe-ne2k_pci.rom';
1764         } elsif ($device eq 'pcnet') {
1765             $romfile = 'pxe-pcnet.rom';
1766         } elsif ($device eq 'rtl8139') {
1767             $romfile = 'pxe-rtl8139.rom';
1768         }
1769         $tmpstr .= ",romfile=$romfile" if $romfile;
1770     }
1771
1772     return $tmpstr;
1773 }
1774
1775 sub print_netdev_full {
1776     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1777
1778     my $i = '';
1779     if ($netid =~ m/^net(\d+)$/) {
1780         $i = int($1);
1781     }
1782
1783     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1784
1785     my $ifname = "tap${vmid}i$i";
1786
1787     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1788     die "interface name '$ifname' is too long (max 15 character)\n"
1789         if length($ifname) >= 16;
1790
1791     my $vhostparam = '';
1792     if (is_native($arch)) {
1793         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1794     }
1795
1796     my $vmname = $conf->{name} || "vm$vmid";
1797
1798     my $netdev = "";
1799     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1800
1801     if ($net->{bridge}) {
1802         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1803             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1804     } else {
1805         $netdev = "type=user,id=$netid,hostname=$vmname";
1806     }
1807
1808     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1809
1810     return $netdev;
1811 }
1812
1813 my $vga_map = {
1814     'cirrus' => 'cirrus-vga',
1815     'std' => 'VGA',
1816     'vmware' => 'vmware-svga',
1817     'virtio' => 'virtio-vga',
1818 };
1819
1820 sub print_vga_device {
1821     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1822
1823     my $type = $vga_map->{$vga->{type}};
1824     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1825         $type = 'virtio-gpu';
1826     }
1827     my $vgamem_mb = $vga->{memory};
1828
1829     my $max_outputs = '';
1830     if ($qxlnum) {
1831         $type = $id ? 'qxl' : 'qxl-vga';
1832
1833         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1834             # set max outputs so linux can have up to 4 qxl displays with one device
1835             if (min_version($machine_version, 4, 1)) {
1836                 $max_outputs = ",max_outputs=4";
1837             }
1838         }
1839     }
1840
1841     die "no devicetype for $vga->{type}\n" if !$type;
1842
1843     my $memory = "";
1844     if ($vgamem_mb) {
1845         if ($vga->{type} eq 'virtio') {
1846             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1847             $memory = ",max_hostmem=$bytes";
1848         } elsif ($qxlnum) {
1849             # from https://www.spice-space.org/multiple-monitors.html
1850             $memory = ",vgamem_mb=$vga->{memory}";
1851             my $ram = $vgamem_mb * 4;
1852             my $vram = $vgamem_mb * 2;
1853             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1854         } else {
1855             $memory = ",vgamem_mb=$vga->{memory}";
1856         }
1857     } elsif ($qxlnum && $id) {
1858         $memory = ",ram_size=67108864,vram_size=33554432";
1859     }
1860
1861     my $edidoff = "";
1862     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1863         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1864     }
1865
1866     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1867     my $vgaid = "vga" . ($id // '');
1868     my $pciaddr;
1869     if ($q35 && $vgaid eq 'vga') {
1870         # the first display uses pcie.0 bus on q35 machines
1871         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1872     } else {
1873         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1874     }
1875
1876     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1877 }
1878
1879 sub parse_number_sets {
1880     my ($set) = @_;
1881     my $res = [];
1882     foreach my $part (split(/;/, $set)) {
1883         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1884             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1885             push @$res, [ $1, $2 ];
1886         } else {
1887             die "invalid range: $part\n";
1888         }
1889     }
1890     return $res;
1891 }
1892
1893 sub parse_numa {
1894     my ($data) = @_;
1895
1896     my $res = parse_property_string($numa_fmt, $data);
1897     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1898     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1899     return $res;
1900 }
1901
1902 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1903 sub parse_net {
1904     my ($data) = @_;
1905
1906     my $res = eval { parse_property_string($net_fmt, $data) };
1907     if ($@) {
1908         warn $@;
1909         return;
1910     }
1911     if (!defined($res->{macaddr})) {
1912         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1913         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1914     }
1915     return $res;
1916 }
1917
1918 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1919 sub parse_ipconfig {
1920     my ($data) = @_;
1921
1922     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1923     if ($@) {
1924         warn $@;
1925         return;
1926     }
1927
1928     if ($res->{gw} && !$res->{ip}) {
1929         warn 'gateway specified without specifying an IP address';
1930         return;
1931     }
1932     if ($res->{gw6} && !$res->{ip6}) {
1933         warn 'IPv6 gateway specified without specifying an IPv6 address';
1934         return;
1935     }
1936     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1937         warn 'gateway specified together with DHCP';
1938         return;
1939     }
1940     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1941         # gw6 + auto/dhcp
1942         warn "IPv6 gateway specified together with $res->{ip6} address";
1943         return;
1944     }
1945
1946     if (!$res->{ip} && !$res->{ip6}) {
1947         return { ip => 'dhcp', ip6 => 'dhcp' };
1948     }
1949
1950     return $res;
1951 }
1952
1953 sub print_net {
1954     my $net = shift;
1955
1956     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1957 }
1958
1959 sub add_random_macs {
1960     my ($settings) = @_;
1961
1962     foreach my $opt (keys %$settings) {
1963         next if $opt !~ m/^net(\d+)$/;
1964         my $net = parse_net($settings->{$opt});
1965         next if !$net;
1966         $settings->{$opt} = print_net($net);
1967     }
1968 }
1969
1970 sub vm_is_volid_owner {
1971     my ($storecfg, $vmid, $volid) = @_;
1972
1973     if ($volid !~  m|^/|) {
1974         my ($path, $owner);
1975         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1976         if ($owner && ($owner == $vmid)) {
1977             return 1;
1978         }
1979     }
1980
1981     return;
1982 }
1983
1984 sub vmconfig_register_unused_drive {
1985     my ($storecfg, $vmid, $conf, $drive) = @_;
1986
1987     if (drive_is_cloudinit($drive)) {
1988         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1989         warn $@ if $@;
1990     } elsif (!drive_is_cdrom($drive)) {
1991         my $volid = $drive->{file};
1992         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1993             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1994         }
1995     }
1996 }
1997
1998 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1999 my $smbios1_fmt = {
2000     uuid => {
2001         type => 'string',
2002         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2003         format_description => 'UUID',
2004         description => "Set SMBIOS1 UUID.",
2005         optional => 1,
2006     },
2007     version => {
2008         type => 'string',
2009         pattern => '[A-Za-z0-9+\/]+={0,2}',
2010         format_description => 'Base64 encoded string',
2011         description => "Set SMBIOS1 version.",
2012         optional => 1,
2013     },
2014     serial => {
2015         type => 'string',
2016         pattern => '[A-Za-z0-9+\/]+={0,2}',
2017         format_description => 'Base64 encoded string',
2018         description => "Set SMBIOS1 serial number.",
2019         optional => 1,
2020     },
2021     manufacturer => {
2022         type => 'string',
2023         pattern => '[A-Za-z0-9+\/]+={0,2}',
2024         format_description => 'Base64 encoded string',
2025         description => "Set SMBIOS1 manufacturer.",
2026         optional => 1,
2027     },
2028     product => {
2029         type => 'string',
2030         pattern => '[A-Za-z0-9+\/]+={0,2}',
2031         format_description => 'Base64 encoded string',
2032         description => "Set SMBIOS1 product ID.",
2033         optional => 1,
2034     },
2035     sku => {
2036         type => 'string',
2037         pattern => '[A-Za-z0-9+\/]+={0,2}',
2038         format_description => 'Base64 encoded string',
2039         description => "Set SMBIOS1 SKU string.",
2040         optional => 1,
2041     },
2042     family => {
2043         type => 'string',
2044         pattern => '[A-Za-z0-9+\/]+={0,2}',
2045         format_description => 'Base64 encoded string',
2046         description => "Set SMBIOS1 family string.",
2047         optional => 1,
2048     },
2049     base64 => {
2050         type => 'boolean',
2051         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2052         optional => 1,
2053     },
2054 };
2055
2056 sub parse_smbios1 {
2057     my ($data) = @_;
2058
2059     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2060     warn $@ if $@;
2061     return $res;
2062 }
2063
2064 sub print_smbios1 {
2065     my ($smbios1) = @_;
2066     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2067 }
2068
2069 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2070
2071 sub parse_watchdog {
2072     my ($value) = @_;
2073
2074     return if !$value;
2075
2076     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2077     warn $@ if $@;
2078     return $res;
2079 }
2080
2081 sub parse_guest_agent {
2082     my ($conf) = @_;
2083
2084     return {} if !defined($conf->{agent});
2085
2086     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2087     warn $@ if $@;
2088
2089     # if the agent is disabled ignore the other potentially set properties
2090     return {} if !$res->{enabled};
2091     return $res;
2092 }
2093
2094 sub get_qga_key {
2095     my ($conf, $key) = @_;
2096     return undef if !defined($conf->{agent});
2097
2098     my $agent = parse_guest_agent($conf);
2099     return $agent->{$key};
2100 }
2101
2102 sub parse_vga {
2103     my ($value) = @_;
2104
2105     return {} if !$value;
2106     my $res = eval { parse_property_string($vga_fmt, $value) };
2107     warn $@ if $@;
2108     return $res;
2109 }
2110
2111 sub parse_rng {
2112     my ($value) = @_;
2113
2114     return if !$value;
2115
2116     my $res = eval { parse_property_string($rng_fmt, $value) };
2117     warn $@ if $@;
2118     return $res;
2119 }
2120
2121 sub parse_meta_info {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub new_meta_info_string {
2132     my () = @_; # for now do not allow to override any value
2133
2134     return PVE::JSONSchema::print_property_string(
2135         {
2136             'creation-qemu' => kvm_user_version(),
2137             ctime => "". int(time()),
2138         },
2139         $meta_info_fmt
2140     );
2141 }
2142
2143 sub qemu_created_version_fixups {
2144     my ($conf, $forcemachine, $kvmver) = @_;
2145
2146     my $meta = parse_meta_info($conf->{meta}) // {};
2147     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2148
2149     # check if we need to apply some handling for VMs that always use the latest machine version but
2150     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2151     # would be required (we do not want to pin machine version for non-windows OS type)
2152     if (
2153         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2154         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2155         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2156         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2157     ) {
2158         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2159         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2160             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2161             # and thus with the predictable interface naming of systemd
2162             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2163         }
2164     }
2165     return;
2166 }
2167
2168 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2169 sub verify_usb_device {
2170     my ($value, $noerr) = @_;
2171
2172     return $value if parse_usb_device($value);
2173
2174     return if $noerr;
2175
2176     die "unable to parse usb device\n";
2177 }
2178
2179 # add JSON properties for create and set function
2180 sub json_config_properties {
2181     my $prop = shift;
2182
2183     my $skip_json_config_opts = {
2184         parent => 1,
2185         snaptime => 1,
2186         vmstate => 1,
2187         runningmachine => 1,
2188         runningcpu => 1,
2189         meta => 1,
2190     };
2191
2192     foreach my $opt (keys %$confdesc) {
2193         next if $skip_json_config_opts->{$opt};
2194         $prop->{$opt} = $confdesc->{$opt};
2195     }
2196
2197     return $prop;
2198 }
2199
2200 # return copy of $confdesc_cloudinit to generate documentation
2201 sub cloudinit_config_properties {
2202
2203     return dclone($confdesc_cloudinit);
2204 }
2205
2206 sub check_type {
2207     my ($key, $value) = @_;
2208
2209     die "unknown setting '$key'\n" if !$confdesc->{$key};
2210
2211     my $type = $confdesc->{$key}->{type};
2212
2213     if (!defined($value)) {
2214         die "got undefined value\n";
2215     }
2216
2217     if ($value =~ m/[\n\r]/) {
2218         die "property contains a line feed\n";
2219     }
2220
2221     if ($type eq 'boolean') {
2222         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2223         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2224         die "type check ('boolean') failed - got '$value'\n";
2225     } elsif ($type eq 'integer') {
2226         return int($1) if $value =~ m/^(\d+)$/;
2227         die "type check ('integer') failed - got '$value'\n";
2228     } elsif ($type eq 'number') {
2229         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2230         die "type check ('number') failed - got '$value'\n";
2231     } elsif ($type eq 'string') {
2232         if (my $fmt = $confdesc->{$key}->{format}) {
2233             PVE::JSONSchema::check_format($fmt, $value);
2234             return $value;
2235         }
2236         $value =~ s/^\"(.*)\"$/$1/;
2237         return $value;
2238     } else {
2239         die "internal error"
2240     }
2241 }
2242
2243 sub destroy_vm {
2244     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2245
2246     my $conf = PVE::QemuConfig->load_config($vmid);
2247
2248     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2249
2250     if ($conf->{template}) {
2251         # check if any base image is still used by a linked clone
2252         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2253                 my ($ds, $drive) = @_;
2254                 return if drive_is_cdrom($drive);
2255
2256                 my $volid = $drive->{file};
2257                 return if !$volid || $volid =~ m|^/|;
2258
2259                 die "base volume '$volid' is still in use by linked cloned\n"
2260                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2261
2262         });
2263     }
2264
2265     my $volids = {};
2266     my $remove_owned_drive = sub {
2267         my ($ds, $drive) = @_;
2268         return if drive_is_cdrom($drive, 1);
2269
2270         my $volid = $drive->{file};
2271         return if !$volid || $volid =~ m|^/|;
2272         return if $volids->{$volid};
2273
2274         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2275         return if !$path || !$owner || ($owner != $vmid);
2276
2277         $volids->{$volid} = 1;
2278         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2279         warn "Could not remove disk '$volid', check manually: $@" if $@;
2280     };
2281
2282     # only remove disks owned by this VM (referenced in the config)
2283     my $include_opts = {
2284         include_unused => 1,
2285         extra_keys => ['vmstate'],
2286     };
2287     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2288
2289     for my $snap (values %{$conf->{snapshots}}) {
2290         next if !defined($snap->{vmstate});
2291         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2292         next if !defined($drive);
2293         $remove_owned_drive->('vmstate', $drive);
2294     }
2295
2296     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2297
2298     if ($purge_unreferenced) { # also remove unreferenced disk
2299         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2300         PVE::Storage::foreach_volid($vmdisks, sub {
2301             my ($volid, $sid, $volname, $d) = @_;
2302             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2303             warn $@ if $@;
2304         });
2305     }
2306
2307     if (defined $replacement_conf) {
2308         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2309     } else {
2310         PVE::QemuConfig->destroy_config($vmid);
2311     }
2312 }
2313
2314 sub parse_vm_config {
2315     my ($filename, $raw, $strict) = @_;
2316
2317     return if !defined($raw);
2318
2319     my $res = {
2320         digest => Digest::SHA::sha1_hex($raw),
2321         snapshots => {},
2322         pending => {},
2323     };
2324
2325     my $handle_error = sub {
2326         my ($msg) = @_;
2327
2328         if ($strict) {
2329             die $msg;
2330         } else {
2331             warn $msg;
2332         }
2333     };
2334
2335     $filename =~ m|/qemu-server/(\d+)\.conf$|
2336         || die "got strange filename '$filename'";
2337
2338     my $vmid = $1;
2339
2340     my $conf = $res;
2341     my $descr;
2342     my $section = '';
2343
2344     my @lines = split(/\n/, $raw);
2345     foreach my $line (@lines) {
2346         next if $line =~ m/^\s*$/;
2347
2348         if ($line =~ m/^\[PENDING\]\s*$/i) {
2349             $section = 'pending';
2350             if (defined($descr)) {
2351                 $descr =~ s/\s+$//;
2352                 $conf->{description} = $descr;
2353             }
2354             $descr = undef;
2355             $conf = $res->{$section} = {};
2356             next;
2357
2358         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2359             $section = $1;
2360             if (defined($descr)) {
2361                 $descr =~ s/\s+$//;
2362                 $conf->{description} = $descr;
2363             }
2364             $descr = undef;
2365             $conf = $res->{snapshots}->{$section} = {};
2366             next;
2367         }
2368
2369         if ($line =~ m/^\#(.*)\s*$/) {
2370             $descr = '' if !defined($descr);
2371             $descr .= PVE::Tools::decode_text($1) . "\n";
2372             next;
2373         }
2374
2375         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2376             $descr = '' if !defined($descr);
2377             $descr .= PVE::Tools::decode_text($2);
2378         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2379             $conf->{snapstate} = $1;
2380         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2381             my $key = $1;
2382             my $value = $2;
2383             $conf->{$key} = $value;
2384         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2385             my $value = $1;
2386             if ($section eq 'pending') {
2387                 $conf->{delete} = $value; # we parse this later
2388             } else {
2389                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2390             }
2391         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2392             my $key = $1;
2393             my $value = $2;
2394             eval { $value = check_type($key, $value); };
2395             if ($@) {
2396                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2397             } else {
2398                 $key = 'ide2' if $key eq 'cdrom';
2399                 my $fmt = $confdesc->{$key}->{format};
2400                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2401                     my $v = parse_drive($key, $value);
2402                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2403                         $v->{file} = $volid;
2404                         $value = print_drive($v);
2405                     } else {
2406                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2407                         next;
2408                     }
2409                 }
2410
2411                 $conf->{$key} = $value;
2412             }
2413         } else {
2414             $handle_error->("vm $vmid - unable to parse config: $line\n");
2415         }
2416     }
2417
2418     if (defined($descr)) {
2419         $descr =~ s/\s+$//;
2420         $conf->{description} = $descr;
2421     }
2422     delete $res->{snapstate}; # just to be sure
2423
2424     return $res;
2425 }
2426
2427 sub write_vm_config {
2428     my ($filename, $conf) = @_;
2429
2430     delete $conf->{snapstate}; # just to be sure
2431
2432     if ($conf->{cdrom}) {
2433         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2434         $conf->{ide2} = $conf->{cdrom};
2435         delete $conf->{cdrom};
2436     }
2437
2438     # we do not use 'smp' any longer
2439     if ($conf->{sockets}) {
2440         delete $conf->{smp};
2441     } elsif ($conf->{smp}) {
2442         $conf->{sockets} = $conf->{smp};
2443         delete $conf->{cores};
2444         delete $conf->{smp};
2445     }
2446
2447     my $used_volids = {};
2448
2449     my $cleanup_config = sub {
2450         my ($cref, $pending, $snapname) = @_;
2451
2452         foreach my $key (keys %$cref) {
2453             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2454                 $key eq 'snapstate' || $key eq 'pending';
2455             my $value = $cref->{$key};
2456             if ($key eq 'delete') {
2457                 die "propertry 'delete' is only allowed in [PENDING]\n"
2458                     if !$pending;
2459                 # fixme: check syntax?
2460                 next;
2461             }
2462             eval { $value = check_type($key, $value); };
2463             die "unable to parse value of '$key' - $@" if $@;
2464
2465             $cref->{$key} = $value;
2466
2467             if (!$snapname && is_valid_drivename($key)) {
2468                 my $drive = parse_drive($key, $value);
2469                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2470             }
2471         }
2472     };
2473
2474     &$cleanup_config($conf);
2475
2476     &$cleanup_config($conf->{pending}, 1);
2477
2478     foreach my $snapname (keys %{$conf->{snapshots}}) {
2479         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2480         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2481     }
2482
2483     # remove 'unusedX' settings if we re-add a volume
2484     foreach my $key (keys %$conf) {
2485         my $value = $conf->{$key};
2486         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2487             delete $conf->{$key};
2488         }
2489     }
2490
2491     my $generate_raw_config = sub {
2492         my ($conf, $pending) = @_;
2493
2494         my $raw = '';
2495
2496         # add description as comment to top of file
2497         if (defined(my $descr = $conf->{description})) {
2498             if ($descr) {
2499                 foreach my $cl (split(/\n/, $descr)) {
2500                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2501                 }
2502             } else {
2503                 $raw .= "#\n" if $pending;
2504             }
2505         }
2506
2507         foreach my $key (sort keys %$conf) {
2508             next if $key =~ /^(digest|description|pending|snapshots)$/;
2509             $raw .= "$key: $conf->{$key}\n";
2510         }
2511         return $raw;
2512     };
2513
2514     my $raw = &$generate_raw_config($conf);
2515
2516     if (scalar(keys %{$conf->{pending}})){
2517         $raw .= "\n[PENDING]\n";
2518         $raw .= &$generate_raw_config($conf->{pending}, 1);
2519     }
2520
2521     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2522         $raw .= "\n[$snapname]\n";
2523         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2524     }
2525
2526     return $raw;
2527 }
2528
2529 sub load_defaults {
2530
2531     my $res = {};
2532
2533     # we use static defaults from our JSON schema configuration
2534     foreach my $key (keys %$confdesc) {
2535         if (defined(my $default = $confdesc->{$key}->{default})) {
2536             $res->{$key} = $default;
2537         }
2538     }
2539
2540     return $res;
2541 }
2542
2543 sub config_list {
2544     my $vmlist = PVE::Cluster::get_vmlist();
2545     my $res = {};
2546     return $res if !$vmlist || !$vmlist->{ids};
2547     my $ids = $vmlist->{ids};
2548     my $nodename = nodename();
2549
2550     foreach my $vmid (keys %$ids) {
2551         my $d = $ids->{$vmid};
2552         next if !$d->{node} || $d->{node} ne $nodename;
2553         next if !$d->{type} || $d->{type} ne 'qemu';
2554         $res->{$vmid}->{exists} = 1;
2555     }
2556     return $res;
2557 }
2558
2559 # test if VM uses local resources (to prevent migration)
2560 sub check_local_resources {
2561     my ($conf, $noerr) = @_;
2562
2563     my @loc_res = ();
2564
2565     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2566     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2567
2568     push @loc_res, "ivshmem" if $conf->{ivshmem};
2569
2570     foreach my $k (keys %$conf) {
2571         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2572         # sockets are safe: they will recreated be on the target side post-migrate
2573         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2574         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2575     }
2576
2577     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2578
2579     return \@loc_res;
2580 }
2581
2582 # check if used storages are available on all nodes (use by migrate)
2583 sub check_storage_availability {
2584     my ($storecfg, $conf, $node) = @_;
2585
2586     PVE::QemuConfig->foreach_volume($conf, sub {
2587         my ($ds, $drive) = @_;
2588
2589         my $volid = $drive->{file};
2590         return if !$volid;
2591
2592         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2593         return if !$sid;
2594
2595         # check if storage is available on both nodes
2596         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2597         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2598
2599         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2600
2601         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2602             if !$scfg->{content}->{$vtype};
2603    });
2604 }
2605
2606 # list nodes where all VM images are available (used by has_feature API)
2607 sub shared_nodes {
2608     my ($conf, $storecfg) = @_;
2609
2610     my $nodelist = PVE::Cluster::get_nodelist();
2611     my $nodehash = { map { $_ => 1 } @$nodelist };
2612     my $nodename = nodename();
2613
2614     PVE::QemuConfig->foreach_volume($conf, sub {
2615         my ($ds, $drive) = @_;
2616
2617         my $volid = $drive->{file};
2618         return if !$volid;
2619
2620         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2621         if ($storeid) {
2622             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2623             if ($scfg->{disable}) {
2624                 $nodehash = {};
2625             } elsif (my $avail = $scfg->{nodes}) {
2626                 foreach my $node (keys %$nodehash) {
2627                     delete $nodehash->{$node} if !$avail->{$node};
2628                 }
2629             } elsif (!$scfg->{shared}) {
2630                 foreach my $node (keys %$nodehash) {
2631                     delete $nodehash->{$node} if $node ne $nodename
2632                 }
2633             }
2634         }
2635     });
2636
2637     return $nodehash
2638 }
2639
2640 sub check_local_storage_availability {
2641     my ($conf, $storecfg) = @_;
2642
2643     my $nodelist = PVE::Cluster::get_nodelist();
2644     my $nodehash = { map { $_ => {} } @$nodelist };
2645
2646     PVE::QemuConfig->foreach_volume($conf, sub {
2647         my ($ds, $drive) = @_;
2648
2649         my $volid = $drive->{file};
2650         return if !$volid;
2651
2652         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2653         if ($storeid) {
2654             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2655
2656             if ($scfg->{disable}) {
2657                 foreach my $node (keys %$nodehash) {
2658                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2659                 }
2660             } elsif (my $avail = $scfg->{nodes}) {
2661                 foreach my $node (keys %$nodehash) {
2662                     if (!$avail->{$node}) {
2663                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2664                     }
2665                 }
2666             }
2667         }
2668     });
2669
2670     foreach my $node (values %$nodehash) {
2671         if (my $unavail = $node->{unavailable_storages}) {
2672             $node->{unavailable_storages} = [ sort keys %$unavail ];
2673         }
2674     }
2675
2676     return $nodehash
2677 }
2678
2679 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2680 sub check_running {
2681     my ($vmid, $nocheck, $node) = @_;
2682
2683     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2684     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2685 }
2686
2687 sub vzlist {
2688
2689     my $vzlist = config_list();
2690
2691     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2692
2693     while (defined(my $de = $fd->read)) {
2694         next if $de !~ m/^(\d+)\.pid$/;
2695         my $vmid = $1;
2696         next if !defined($vzlist->{$vmid});
2697         if (my $pid = check_running($vmid)) {
2698             $vzlist->{$vmid}->{pid} = $pid;
2699         }
2700     }
2701
2702     return $vzlist;
2703 }
2704
2705 our $vmstatus_return_properties = {
2706     vmid => get_standard_option('pve-vmid'),
2707     status => {
2708         description => "Qemu process status.",
2709         type => 'string',
2710         enum => ['stopped', 'running'],
2711     },
2712     maxmem => {
2713         description => "Maximum memory in bytes.",
2714         type => 'integer',
2715         optional => 1,
2716         renderer => 'bytes',
2717     },
2718     maxdisk => {
2719         description => "Root disk size in bytes.",
2720         type => 'integer',
2721         optional => 1,
2722         renderer => 'bytes',
2723     },
2724     name => {
2725         description => "VM name.",
2726         type => 'string',
2727         optional => 1,
2728     },
2729     qmpstatus => {
2730         description => "Qemu QMP agent status.",
2731         type => 'string',
2732         optional => 1,
2733     },
2734     pid => {
2735         description => "PID of running qemu process.",
2736         type => 'integer',
2737         optional => 1,
2738     },
2739     uptime => {
2740         description => "Uptime.",
2741         type => 'integer',
2742         optional => 1,
2743         renderer => 'duration',
2744     },
2745     cpus => {
2746         description => "Maximum usable CPUs.",
2747         type => 'number',
2748         optional => 1,
2749     },
2750     lock => {
2751         description => "The current config lock, if any.",
2752         type => 'string',
2753         optional => 1,
2754     },
2755     tags => {
2756         description  => "The current configured tags, if any",
2757         type => 'string',
2758         optional => 1,
2759     },
2760     'running-machine' => {
2761         description => "The currently running machine type (if running).",
2762         type => 'string',
2763         optional => 1,
2764     },
2765     'running-qemu' => {
2766         description => "The currently running QEMU version (if running).",
2767         type => 'string',
2768         optional => 1,
2769     },
2770 };
2771
2772 my $last_proc_pid_stat;
2773
2774 # get VM status information
2775 # This must be fast and should not block ($full == false)
2776 # We only query KVM using QMP if $full == true (this can be slow)
2777 sub vmstatus {
2778     my ($opt_vmid, $full) = @_;
2779
2780     my $res = {};
2781
2782     my $storecfg = PVE::Storage::config();
2783
2784     my $list = vzlist();
2785     my $defaults = load_defaults();
2786
2787     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2788
2789     my $cpucount = $cpuinfo->{cpus} || 1;
2790
2791     foreach my $vmid (keys %$list) {
2792         next if $opt_vmid && ($vmid ne $opt_vmid);
2793
2794         my $conf = PVE::QemuConfig->load_config($vmid);
2795
2796         my $d = { vmid => int($vmid) };
2797         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2798
2799         # fixme: better status?
2800         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2801
2802         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2803         if (defined($size)) {
2804             $d->{disk} = 0; # no info available
2805             $d->{maxdisk} = $size;
2806         } else {
2807             $d->{disk} = 0;
2808             $d->{maxdisk} = 0;
2809         }
2810
2811         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2812             * ($conf->{cores} || $defaults->{cores});
2813         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2814         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2815
2816         $d->{name} = $conf->{name} || "VM $vmid";
2817         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2818             : $defaults->{memory}*(1024*1024);
2819
2820         if ($conf->{balloon}) {
2821             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2822             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2823                 : $defaults->{shares};
2824         }
2825
2826         $d->{uptime} = 0;
2827         $d->{cpu} = 0;
2828         $d->{mem} = 0;
2829
2830         $d->{netout} = 0;
2831         $d->{netin} = 0;
2832
2833         $d->{diskread} = 0;
2834         $d->{diskwrite} = 0;
2835
2836         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2837
2838         $d->{serial} = 1 if conf_has_serial($conf);
2839         $d->{lock} = $conf->{lock} if $conf->{lock};
2840         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2841
2842         $res->{$vmid} = $d;
2843     }
2844
2845     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2846     foreach my $dev (keys %$netdev) {
2847         next if $dev !~ m/^tap([1-9]\d*)i/;
2848         my $vmid = $1;
2849         my $d = $res->{$vmid};
2850         next if !$d;
2851
2852         $d->{netout} += $netdev->{$dev}->{receive};
2853         $d->{netin} += $netdev->{$dev}->{transmit};
2854
2855         if ($full) {
2856             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2857             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2858         }
2859
2860     }
2861
2862     my $ctime = gettimeofday;
2863
2864     foreach my $vmid (keys %$list) {
2865
2866         my $d = $res->{$vmid};
2867         my $pid = $d->{pid};
2868         next if !$pid;
2869
2870         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2871         next if !$pstat; # not running
2872
2873         my $used = $pstat->{utime} + $pstat->{stime};
2874
2875         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2876
2877         if ($pstat->{vsize}) {
2878             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2879         }
2880
2881         my $old = $last_proc_pid_stat->{$pid};
2882         if (!$old) {
2883             $last_proc_pid_stat->{$pid} = {
2884                 time => $ctime,
2885                 used => $used,
2886                 cpu => 0,
2887             };
2888             next;
2889         }
2890
2891         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2892
2893         if ($dtime > 1000) {
2894             my $dutime = $used -  $old->{used};
2895
2896             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2897             $last_proc_pid_stat->{$pid} = {
2898                 time => $ctime,
2899                 used => $used,
2900                 cpu => $d->{cpu},
2901             };
2902         } else {
2903             $d->{cpu} = $old->{cpu};
2904         }
2905     }
2906
2907     return $res if !$full;
2908
2909     my $qmpclient = PVE::QMPClient->new();
2910
2911     my $ballooncb = sub {
2912         my ($vmid, $resp) = @_;
2913
2914         my $info = $resp->{'return'};
2915         return if !$info->{max_mem};
2916
2917         my $d = $res->{$vmid};
2918
2919         # use memory assigned to VM
2920         $d->{maxmem} = $info->{max_mem};
2921         $d->{balloon} = $info->{actual};
2922
2923         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2924             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2925             $d->{freemem} = $info->{free_mem};
2926         }
2927
2928         $d->{ballooninfo} = $info;
2929     };
2930
2931     my $blockstatscb = sub {
2932         my ($vmid, $resp) = @_;
2933         my $data = $resp->{'return'} || [];
2934         my $totalrdbytes = 0;
2935         my $totalwrbytes = 0;
2936
2937         for my $blockstat (@$data) {
2938             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2939             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2940
2941             $blockstat->{device} =~ s/drive-//;
2942             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2943         }
2944         $res->{$vmid}->{diskread} = $totalrdbytes;
2945         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2946     };
2947
2948     my $machinecb = sub {
2949         my ($vmid, $resp) = @_;
2950         my $data = $resp->{'return'} || [];
2951
2952         $res->{$vmid}->{'running-machine'} =
2953             PVE::QemuServer::Machine::current_from_query_machines($data);
2954     };
2955
2956     my $versioncb = sub {
2957         my ($vmid, $resp) = @_;
2958         my $data = $resp->{'return'} // {};
2959         my $version = 'unknown';
2960
2961         if (my $v = $data->{qemu}) {
2962             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2963         }
2964
2965         $res->{$vmid}->{'running-qemu'} = $version;
2966     };
2967
2968     my $statuscb = sub {
2969         my ($vmid, $resp) = @_;
2970
2971         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2972         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2973         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2974         # this fails if ballon driver is not loaded, so this must be
2975         # the last commnand (following command are aborted if this fails).
2976         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2977
2978         my $status = 'unknown';
2979         if (!defined($status = $resp->{'return'}->{status})) {
2980             warn "unable to get VM status\n";
2981             return;
2982         }
2983
2984         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2985     };
2986
2987     foreach my $vmid (keys %$list) {
2988         next if $opt_vmid && ($vmid ne $opt_vmid);
2989         next if !$res->{$vmid}->{pid}; # not running
2990         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2991     }
2992
2993     $qmpclient->queue_execute(undef, 2);
2994
2995     foreach my $vmid (keys %$list) {
2996         next if $opt_vmid && ($vmid ne $opt_vmid);
2997         next if !$res->{$vmid}->{pid}; #not running
2998
2999         # we can't use the $qmpclient since it might have already aborted on
3000         # 'query-balloon', but this might also fail for older versions...
3001         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3002         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3003     }
3004
3005     foreach my $vmid (keys %$list) {
3006         next if $opt_vmid && ($vmid ne $opt_vmid);
3007         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3008     }
3009
3010     return $res;
3011 }
3012
3013 sub conf_has_serial {
3014     my ($conf) = @_;
3015
3016     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3017         if ($conf->{"serial$i"}) {
3018             return 1;
3019         }
3020     }
3021
3022     return 0;
3023 }
3024
3025 sub conf_has_audio {
3026     my ($conf, $id) = @_;
3027
3028     $id //= 0;
3029     my $audio = $conf->{"audio$id"};
3030     return if !defined($audio);
3031
3032     my $audioproperties = parse_property_string($audio_fmt, $audio);
3033     my $audiodriver = $audioproperties->{driver} // 'spice';
3034
3035     return {
3036         dev => $audioproperties->{device},
3037         dev_id => "audiodev$id",
3038         backend => $audiodriver,
3039         backend_id => "$audiodriver-backend${id}",
3040     };
3041 }
3042
3043 sub audio_devs {
3044     my ($audio, $audiopciaddr, $machine_version) = @_;
3045
3046     my $devs = [];
3047
3048     my $id = $audio->{dev_id};
3049     my $audiodev = "";
3050     if (min_version($machine_version, 4, 2)) {
3051         $audiodev = ",audiodev=$audio->{backend_id}";
3052     }
3053
3054     if ($audio->{dev} eq 'AC97') {
3055         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3056     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3057         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3058         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3059         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3060     } else {
3061         die "unkown audio device '$audio->{dev}', implement me!";
3062     }
3063
3064     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3065
3066     return $devs;
3067 }
3068
3069 sub get_tpm_paths {
3070     my ($vmid) = @_;
3071     return {
3072         socket => "/var/run/qemu-server/$vmid.swtpm",
3073         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3074     };
3075 }
3076
3077 sub add_tpm_device {
3078     my ($vmid, $devices, $conf) = @_;
3079
3080     return if !$conf->{tpmstate0};
3081
3082     my $paths = get_tpm_paths($vmid);
3083
3084     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3085     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3086     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3087 }
3088
3089 sub start_swtpm {
3090     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3091
3092     return if !$tpmdrive;
3093
3094     my $state;
3095     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3096     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3097     if ($storeid) {
3098         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3099     } else {
3100         $state = $tpm->{file};
3101     }
3102
3103     my $paths = get_tpm_paths($vmid);
3104
3105     # during migration, we will get state from remote
3106     #
3107     if (!$migration) {
3108         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3109         my $setup_cmd = [
3110             "swtpm_setup",
3111             "--tpmstate",
3112             "file://$state",
3113             "--createek",
3114             "--create-ek-cert",
3115             "--create-platform-cert",
3116             "--lock-nvram",
3117             "--config",
3118             "/etc/swtpm_setup.conf", # do not use XDG configs
3119             "--runas",
3120             "0", # force creation as root, error if not possible
3121             "--not-overwrite", # ignore existing state, do not modify
3122         ];
3123
3124         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3125         # TPM 2.0 supports ECC crypto, use if possible
3126         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3127
3128         run_command($setup_cmd, outfunc => sub {
3129             print "swtpm_setup: $1\n";
3130         });
3131     }
3132
3133     my $emulator_cmd = [
3134         "swtpm",
3135         "socket",
3136         "--tpmstate",
3137         "backend-uri=file://$state,mode=0600",
3138         "--ctrl",
3139         "type=unixio,path=$paths->{socket},mode=0600",
3140         "--pid",
3141         "file=$paths->{pid}",
3142         "--terminate", # terminate on QEMU disconnect
3143         "--daemon",
3144     ];
3145     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3146     run_command($emulator_cmd, outfunc => sub { print $1; });
3147
3148     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3149     while (! -e $paths->{pid}) {
3150         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3151         usleep(50_000);
3152     }
3153
3154     # return untainted PID of swtpm daemon so it can be killed on error
3155     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3156     return $1;
3157 }
3158
3159 sub vga_conf_has_spice {
3160     my ($vga) = @_;
3161
3162     my $vgaconf = parse_vga($vga);
3163     my $vgatype = $vgaconf->{type};
3164     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3165
3166     return $1 || 1;
3167 }
3168
3169 sub is_native($) {
3170     my ($arch) = @_;
3171     return get_host_arch() eq $arch;
3172 }
3173
3174 sub get_vm_arch {
3175     my ($conf) = @_;
3176     return $conf->{arch} // get_host_arch();
3177 }
3178
3179 my $default_machines = {
3180     x86_64 => 'pc',
3181     aarch64 => 'virt',
3182 };
3183
3184 sub get_installed_machine_version {
3185     my ($kvmversion) = @_;
3186     $kvmversion = kvm_user_version() if !defined($kvmversion);
3187     $kvmversion =~ m/^(\d+\.\d+)/;
3188     return $1;
3189 }
3190
3191 sub windows_get_pinned_machine_version {
3192     my ($machine, $base_version, $kvmversion) = @_;
3193
3194     my $pin_version = $base_version;
3195     if (!defined($base_version) ||
3196         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3197     ) {
3198         $pin_version = get_installed_machine_version($kvmversion);
3199     }
3200     if (!$machine || $machine eq 'pc') {
3201         $machine = "pc-i440fx-$pin_version";
3202     } elsif ($machine eq 'q35') {
3203         $machine = "pc-q35-$pin_version";
3204     } elsif ($machine eq 'virt') {
3205         $machine = "virt-$pin_version";
3206     } else {
3207         warn "unknown machine type '$machine', not touching that!\n";
3208     }
3209
3210     return $machine;
3211 }
3212
3213 sub get_vm_machine {
3214     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3215
3216     my $machine = $forcemachine || $conf->{machine};
3217
3218     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3219         $kvmversion //= kvm_user_version();
3220         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3221         # layout which confuses windows quite a bit and may result in various regressions..
3222         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3223         if (windows_version($conf->{ostype})) {
3224             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3225         }
3226         $arch //= 'x86_64';
3227         $machine ||= $default_machines->{$arch};
3228         if ($add_pve_version) {
3229             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3230             $machine .= "+pve$pvever";
3231         }
3232     }
3233
3234     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3235         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3236         $machine = $1 if $is_pxe;
3237
3238         # for version-pinned machines that do not include a pve-version (e.g.
3239         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3240         $machine .= '+pve0';
3241
3242         $machine .= '.pxe' if $is_pxe;
3243     }
3244
3245     return $machine;
3246 }
3247
3248 sub get_ovmf_files($$$) {
3249     my ($arch, $efidisk, $smm) = @_;
3250
3251     my $types = $OVMF->{$arch}
3252         or die "no OVMF images known for architecture '$arch'\n";
3253
3254     my $type = 'default';
3255     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3256         $type = $smm ? "4m" : "4m-no-smm";
3257         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3258     }
3259
3260     return $types->{$type}->@*;
3261 }
3262
3263 my $Arch2Qemu = {
3264     aarch64 => '/usr/bin/qemu-system-aarch64',
3265     x86_64 => '/usr/bin/qemu-system-x86_64',
3266 };
3267 sub get_command_for_arch($) {
3268     my ($arch) = @_;
3269     return '/usr/bin/kvm' if is_native($arch);
3270
3271     my $cmd = $Arch2Qemu->{$arch}
3272         or die "don't know how to emulate architecture '$arch'\n";
3273     return $cmd;
3274 }
3275
3276 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3277 # to use in a QEMU command line (-cpu element), first array_intersect the result
3278 # of query_supported_ with query_understood_. This is necessary because:
3279 #
3280 # a) query_understood_ returns flags the host cannot use and
3281 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3282 #    flags, but CPU settings - with most of them being flags. Those settings
3283 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3284 #
3285 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3286 # expensive. If you need the value returned from this, you can get it much
3287 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3288 # $accel being 'kvm' or 'tcg'.
3289 #
3290 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3291 # changes, automatically populating pmxcfs.
3292 #
3293 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3294 # since kvm and tcg machines support different flags
3295 #
3296 sub query_supported_cpu_flags {
3297     my ($arch) = @_;
3298
3299     $arch //= get_host_arch();
3300     my $default_machine = $default_machines->{$arch};
3301
3302     my $flags = {};
3303
3304     # FIXME: Once this is merged, the code below should work for ARM as well:
3305     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3306     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3307         $arch eq "aarch64";
3308
3309     my $kvm_supported = defined(kvm_version());
3310     my $qemu_cmd = get_command_for_arch($arch);
3311     my $fakevmid = -1;
3312     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3313
3314     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3315     my $query_supported_run_qemu = sub {
3316         my ($kvm) = @_;
3317
3318         my $flags = {};
3319         my $cmd = [
3320             $qemu_cmd,
3321             '-machine', $default_machine,
3322             '-display', 'none',
3323             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3324             '-mon', 'chardev=qmp,mode=control',
3325             '-pidfile', $pidfile,
3326             '-S', '-daemonize'
3327         ];
3328
3329         if (!$kvm) {
3330             push @$cmd, '-accel', 'tcg';
3331         }
3332
3333         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3334         die "QEMU flag querying VM exited with code " . $rc if $rc;
3335
3336         eval {
3337             my $cmd_result = mon_cmd(
3338                 $fakevmid,
3339                 'query-cpu-model-expansion',
3340                 type => 'full',
3341                 model => { name => 'host' }
3342             );
3343
3344             my $props = $cmd_result->{model}->{props};
3345             foreach my $prop (keys %$props) {
3346                 next if $props->{$prop} ne '1';
3347                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3348                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3349                 # We only keep those with underscores, to match /proc/cpuinfo
3350                 $prop =~ s/\.|-/_/g;
3351                 $flags->{$prop} = 1;
3352             }
3353         };
3354         my $err = $@;
3355
3356         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3357         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3358
3359         die $err if $err;
3360
3361         return [ sort keys %$flags ];
3362     };
3363
3364     # We need to query QEMU twice, since KVM and TCG have different supported flags
3365     PVE::QemuConfig->lock_config($fakevmid, sub {
3366         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3367         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3368
3369         if ($kvm_supported) {
3370             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3371             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3372         }
3373     });
3374
3375     return $flags;
3376 }
3377
3378 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3379 my $understood_cpu_flag_dir = "/usr/share/kvm";
3380 sub query_understood_cpu_flags {
3381     my $arch = get_host_arch();
3382     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3383
3384     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3385         if ! -e $filepath;
3386
3387     my $raw = file_get_contents($filepath);
3388     $raw =~ s/^\s+|\s+$//g;
3389     my @flags = split(/\s+/, $raw);
3390
3391     return \@flags;
3392 }
3393
3394 my sub get_cpuunits {
3395     my ($conf) = @_;
3396     my $is_cgroupv2 = PVE::CGroup::cgroup_mode() == 2;
3397
3398     my $cpuunits = $conf->{cpuunits};
3399     return $is_cgroupv2 ? 100 : 1024 if !defined($cpuunits);
3400
3401     if ($is_cgroupv2) {
3402         $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
3403     } else {
3404         $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
3405     }
3406     return $cpuunits;
3407 }
3408
3409 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3410 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3411 my sub should_disable_smm {
3412     my ($conf, $vga) = @_;
3413
3414     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3415         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3416 }
3417
3418 sub config_to_command {
3419     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3420         $pbs_backing) = @_;
3421
3422     my $cmd = [];
3423     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3424     my $devices = [];
3425     my $bridges = {};
3426     my $ostype = $conf->{ostype};
3427     my $winversion = windows_version($ostype);
3428     my $kvm = $conf->{kvm};
3429     my $nodename = nodename();
3430
3431     my $arch = get_vm_arch($conf);
3432     my $kvm_binary = get_command_for_arch($arch);
3433     my $kvmver = kvm_user_version($kvm_binary);
3434
3435     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3436         $kvmver //= "undefined";
3437         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3438     }
3439
3440     my $add_pve_version = min_version($kvmver, 4, 1);
3441
3442     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3443     my $machine_version = extract_version($machine_type, $kvmver);
3444     $kvm //= 1 if is_native($arch);
3445
3446     $machine_version =~ m/(\d+)\.(\d+)/;
3447     my ($machine_major, $machine_minor) = ($1, $2);
3448
3449     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3450         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3451     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3452         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3453             ." please upgrade node '$nodename'\n"
3454     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3455         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3456         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3457             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3458             ." node '$nodename'\n";
3459     }
3460
3461     # if a specific +pve version is required for a feature, use $version_guard
3462     # instead of min_version to allow machines to be run with the minimum
3463     # required version
3464     my $required_pve_version = 0;
3465     my $version_guard = sub {
3466         my ($major, $minor, $pve) = @_;
3467         return 0 if !min_version($machine_version, $major, $minor, $pve);
3468         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3469         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3470         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3471         return 1;
3472     };
3473
3474     if ($kvm && !defined kvm_version()) {
3475         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3476             ." or enable in BIOS.\n";
3477     }
3478
3479     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3480     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3481     my $use_old_bios_files = undef;
3482     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3483
3484     my $cpuunits = get_cpuunits($conf);
3485
3486     push @$cmd, $kvm_binary;
3487
3488     push @$cmd, '-id', $vmid;
3489
3490     my $vmname = $conf->{name} || "vm$vmid";
3491
3492     push @$cmd, '-name', $vmname;
3493
3494     push @$cmd, '-no-shutdown';
3495
3496     my $use_virtio = 0;
3497
3498     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3499     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3500     push @$cmd, '-mon', "chardev=qmp,mode=control";
3501
3502     if (min_version($machine_version, 2, 12)) {
3503         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3504         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3505     }
3506
3507     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3508
3509     push @$cmd, '-daemonize';
3510
3511     if ($conf->{smbios1}) {
3512         my $smbios_conf = parse_smbios1($conf->{smbios1});
3513         if ($smbios_conf->{base64}) {
3514             # Do not pass base64 flag to qemu
3515             delete $smbios_conf->{base64};
3516             my $smbios_string = "";
3517             foreach my $key (keys %$smbios_conf) {
3518                 my $value;
3519                 if ($key eq "uuid") {
3520                     $value = $smbios_conf->{uuid}
3521                 } else {
3522                     $value = decode_base64($smbios_conf->{$key});
3523                 }
3524                 # qemu accepts any binary data, only commas need escaping by double comma
3525                 $value =~ s/,/,,/g;
3526                 $smbios_string .= "," . $key . "=" . $value if $value;
3527             }
3528             push @$cmd, '-smbios', "type=1" . $smbios_string;
3529         } else {
3530             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3531         }
3532     }
3533
3534     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3535         my $d;
3536         if (my $efidisk = $conf->{efidisk0}) {
3537             $d = parse_drive('efidisk0', $efidisk);
3538         }
3539
3540         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3541         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3542
3543         my ($path, $format);
3544         my $read_only_str = '';
3545         if ($d) {
3546             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3547             $format = $d->{format};
3548             if ($storeid) {
3549                 $path = PVE::Storage::path($storecfg, $d->{file});
3550                 if (!defined($format)) {
3551                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3552                     $format = qemu_img_format($scfg, $volname);
3553                 }
3554             } else {
3555                 $path = $d->{file};
3556                 die "efidisk format must be specified\n"
3557                     if !defined($format);
3558             }
3559
3560             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3561         } else {
3562             warn "no efidisk configured! Using temporary efivars disk.\n";
3563             $path = "/tmp/$vmid-ovmf.fd";
3564             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3565             $format = 'raw';
3566         }
3567
3568         my $size_str = "";
3569
3570         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3571             $size_str = ",size=" . (-s $ovmf_vars);
3572         }
3573
3574         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3575         my $cache = "";
3576         if ($path =~ m/^rbd:/) {
3577                 $cache = ',cache=writeback';
3578                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3579         }
3580
3581         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3582         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3583     }
3584
3585     if ($q35) { # tell QEMU to load q35 config early
3586         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3587         if (min_version($machine_version, 4, 0)) {
3588             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3589         } else {
3590             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3591         }
3592     }
3593
3594     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3595         push @$cmd, $fixups->@*;
3596     }
3597
3598     if ($conf->{vmgenid}) {
3599         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3600     }
3601
3602     # add usb controllers
3603     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3604         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3605     push @$devices, @usbcontrollers if @usbcontrollers;
3606     my $vga = parse_vga($conf->{vga});
3607
3608     my $qxlnum = vga_conf_has_spice($conf->{vga});
3609     $vga->{type} = 'qxl' if $qxlnum;
3610
3611     if (!$vga->{type}) {
3612         if ($arch eq 'aarch64') {
3613             $vga->{type} = 'virtio';
3614         } elsif (min_version($machine_version, 2, 9)) {
3615             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3616         } else {
3617             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3618         }
3619     }
3620
3621     # enable absolute mouse coordinates (needed by vnc)
3622     my $tablet = $conf->{tablet};
3623     if (!defined($tablet)) {
3624         $tablet = $defaults->{tablet};
3625         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3626         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3627     }
3628
3629     if ($tablet) {
3630         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3631         my $kbd = print_keyboarddevice_full($conf, $arch);
3632         push @$devices, '-device', $kbd if defined($kbd);
3633     }
3634
3635     my $bootorder = device_bootorder($conf);
3636
3637     # host pci device passthrough
3638     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3639         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3640
3641     # usb devices
3642     my $usb_dev_features = {};
3643     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3644
3645     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3646         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3647     push @$devices, @usbdevices if @usbdevices;
3648
3649     # serial devices
3650     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3651         my $path = $conf->{"serial$i"} or next;
3652         if ($path eq 'socket') {
3653             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3654             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3655             # On aarch64, serial0 is the UART device. Qemu only allows
3656             # connecting UART devices via the '-serial' command line, as
3657             # the device has a fixed slot on the hardware...
3658             if ($arch eq 'aarch64' && $i == 0) {
3659                 push @$devices, '-serial', "chardev:serial$i";
3660             } else {
3661                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3662             }
3663         } else {
3664             die "no such serial device\n" if ! -c $path;
3665             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3666             push @$devices, '-device', "isa-serial,chardev=serial$i";
3667         }
3668     }
3669
3670     # parallel devices
3671     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3672         if (my $path = $conf->{"parallel$i"}) {
3673             die "no such parallel device\n" if ! -c $path;
3674             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3675             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3676             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3677         }
3678     }
3679
3680     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3681         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3682         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3683         push @$devices, @$audio_devs;
3684     }
3685
3686     add_tpm_device($vmid, $devices, $conf);
3687
3688     my $sockets = 1;
3689     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3690     $sockets = $conf->{sockets} if  $conf->{sockets};
3691
3692     my $cores = $conf->{cores} || 1;
3693
3694     my $maxcpus = $sockets * $cores;
3695
3696     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3697
3698     my $allowed_vcpus = $cpuinfo->{cpus};
3699
3700     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3701
3702     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3703         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3704         for (my $i = 2; $i <= $vcpus; $i++)  {
3705             my $cpustr = print_cpu_device($conf,$i);
3706             push @$cmd, '-device', $cpustr;
3707         }
3708
3709     } else {
3710
3711         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3712     }
3713     push @$cmd, '-nodefaults';
3714
3715     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3716
3717     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3718
3719     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3720
3721     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3722         push @$devices, '-device', print_vga_device(
3723             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3724         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3725         push @$cmd,  '-vnc', "unix:$socket,password=on";
3726     } else {
3727         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3728         push @$cmd, '-nographic';
3729     }
3730
3731     # time drift fix
3732     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3733     my $useLocaltime = $conf->{localtime};
3734
3735     if ($winversion >= 5) { # windows
3736         $useLocaltime = 1 if !defined($conf->{localtime});
3737
3738         # use time drift fix when acpi is enabled
3739         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3740             $tdf = 1 if !defined($conf->{tdf});
3741         }
3742     }
3743
3744     if ($winversion >= 6) {
3745         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3746         push @$cmd, '-no-hpet';
3747     }
3748
3749     push @$rtcFlags, 'driftfix=slew' if $tdf;
3750
3751     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3752         push @$rtcFlags, "base=$conf->{startdate}";
3753     } elsif ($useLocaltime) {
3754         push @$rtcFlags, 'base=localtime';
3755     }
3756
3757     if ($forcecpu) {
3758         push @$cmd, '-cpu', $forcecpu;
3759     } else {
3760         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3761     }
3762
3763     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3764
3765     push @$cmd, '-S' if $conf->{freeze};
3766
3767     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3768
3769     my $guest_agent = parse_guest_agent($conf);
3770
3771     if ($guest_agent->{enabled}) {
3772         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3773         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3774
3775         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3776             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3777             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3778             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3779         } elsif ($guest_agent->{type} eq 'isa') {
3780             push @$devices, '-device', "isa-serial,chardev=qga0";
3781         }
3782     }
3783
3784     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3785     if ($rng && $version_guard->(4, 1, 2)) {
3786         check_rng_source($rng->{source});
3787
3788         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3789         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3790         my $limiter_str = "";
3791         if ($max_bytes) {
3792             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3793         }
3794
3795         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3796         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3797         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3798     }
3799
3800     my $spice_port;
3801
3802     if ($qxlnum) {
3803         if ($qxlnum > 1) {
3804             if ($winversion){
3805                 for (my $i = 1; $i < $qxlnum; $i++){
3806                     push @$devices, '-device', print_vga_device(
3807                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3808                 }
3809             } else {
3810                 # assume other OS works like Linux
3811                 my ($ram, $vram) = ("134217728", "67108864");
3812                 if ($vga->{memory}) {
3813                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3814                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3815                 }
3816                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3817                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3818             }
3819         }
3820
3821         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3822
3823         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3824         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3825         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3826
3827         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3828         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3829         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3830
3831         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3832         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3833
3834         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3835         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3836         if ($spice_enhancement->{foldersharing}) {
3837             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3838             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3839         }
3840
3841         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3842         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3843             if $spice_enhancement->{videostreaming};
3844
3845         push @$devices, '-spice', "$spice_opts";
3846     }
3847
3848     # enable balloon by default, unless explicitly disabled
3849     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3850         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3851         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3852     }
3853
3854     if ($conf->{watchdog}) {
3855         my $wdopts = parse_watchdog($conf->{watchdog});
3856         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3857         my $watchdog = $wdopts->{model} || 'i6300esb';
3858         push @$devices, '-device', "$watchdog$pciaddr";
3859         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3860     }
3861
3862     my $vollist = [];
3863     my $scsicontroller = {};
3864     my $ahcicontroller = {};
3865     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3866
3867     # Add iscsi initiator name if available
3868     if (my $initiator = get_initiator_name()) {
3869         push @$devices, '-iscsi', "initiator-name=$initiator";
3870     }
3871
3872     PVE::QemuConfig->foreach_volume($conf, sub {
3873         my ($ds, $drive) = @_;
3874
3875         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3876             check_volume_storage_type($storecfg, $drive->{file});
3877             push @$vollist, $drive->{file};
3878         }
3879
3880         # ignore efidisk here, already added in bios/fw handling code above
3881         return if $drive->{interface} eq 'efidisk';
3882         # similar for TPM
3883         return if $drive->{interface} eq 'tpmstate';
3884
3885         $use_virtio = 1 if $ds =~ m/^virtio/;
3886
3887         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3888
3889         if ($drive->{interface} eq 'virtio'){
3890            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3891         }
3892
3893         if ($drive->{interface} eq 'scsi') {
3894
3895             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3896
3897             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3898                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3899
3900             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3901             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3902
3903             my $iothread = '';
3904             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3905                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3906                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3907             } elsif ($drive->{iothread}) {
3908                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3909             }
3910
3911             my $queues = '';
3912             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3913                 $queues = ",num_queues=$drive->{queues}";
3914             }
3915
3916             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3917                 if !$scsicontroller->{$controller};
3918             $scsicontroller->{$controller}=1;
3919         }
3920
3921         if ($drive->{interface} eq 'sata') {
3922             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3923             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3924             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3925                 if !$ahcicontroller->{$controller};
3926             $ahcicontroller->{$controller}=1;
3927         }
3928
3929         my $pbs_conf = $pbs_backing->{$ds};
3930         my $pbs_name = undef;
3931         if ($pbs_conf) {
3932             $pbs_name = "drive-$ds-pbs";
3933             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3934         }
3935
3936         my $drive_cmd = print_drive_commandline_full(
3937             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3938
3939         # extra protection for templates, but SATA and IDE don't support it..
3940         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3941
3942         push @$devices, '-drive',$drive_cmd;
3943         push @$devices, '-device', print_drivedevice_full(
3944             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3945     });
3946
3947     for (my $i = 0; $i < $MAX_NETS; $i++) {
3948         my $netname = "net$i";
3949
3950         next if !$conf->{$netname};
3951         my $d = parse_net($conf->{$netname});
3952         next if !$d;
3953
3954         $use_virtio = 1 if $d->{model} eq 'virtio';
3955
3956         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3957
3958         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3959         push @$devices, '-netdev', $netdevfull;
3960
3961         my $netdevicefull = print_netdevice_full(
3962             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3963
3964         push @$devices, '-device', $netdevicefull;
3965     }
3966
3967     if ($conf->{ivshmem}) {
3968         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3969
3970         my $bus;
3971         if ($q35) {
3972             $bus = print_pcie_addr("ivshmem");
3973         } else {
3974             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3975         }
3976
3977         my $ivshmem_name = $ivshmem->{name} // $vmid;
3978         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3979
3980         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3981         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3982             .",size=$ivshmem->{size}M";
3983     }
3984
3985     # pci.4 is nested in pci.1
3986     $bridges->{1} = 1 if $bridges->{4};
3987
3988     if (!$q35) { # add pci bridges
3989         if (min_version($machine_version, 2, 3)) {
3990            $bridges->{1} = 1;
3991            $bridges->{2} = 1;
3992         }
3993         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3994     }
3995
3996     for my $k (sort {$b cmp $a} keys %$bridges) {
3997         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3998
3999         my $k_name = $k;
4000         if ($k == 2 && $legacy_igd) {
4001             $k_name = "$k-igd";
4002         }
4003         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4004         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4005
4006         if ($q35) { # add after -readconfig pve-q35.cfg
4007             splice @$devices, 2, 0, '-device', $devstr;
4008         } else {
4009             unshift @$devices, '-device', $devstr if $k > 0;
4010         }
4011     }
4012
4013     if (!$kvm) {
4014         push @$machineFlags, 'accel=tcg';
4015     }
4016
4017     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4018
4019     my $machine_type_min = $machine_type;
4020     if ($add_pve_version) {
4021         $machine_type_min =~ s/\+pve\d+$//;
4022         $machine_type_min .= "+pve$required_pve_version";
4023     }
4024     push @$machineFlags, "type=${machine_type_min}";
4025
4026     push @$cmd, @$devices;
4027     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4028     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4029     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4030
4031     if (my $vmstate = $conf->{vmstate}) {
4032         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4033         push @$vollist, $vmstate;
4034         push @$cmd, '-loadstate', $statepath;
4035         print "activating and using '$vmstate' as vmstate\n";
4036     }
4037
4038     if (PVE::QemuConfig->is_template($conf)) {
4039         # needed to workaround base volumes being read-only
4040         push @$cmd, '-snapshot';
4041     }
4042
4043     # add custom args
4044     if ($conf->{args}) {
4045         my $aa = PVE::Tools::split_args($conf->{args});
4046         push @$cmd, @$aa;
4047     }
4048
4049     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4050 }
4051
4052 sub check_rng_source {
4053     my ($source) = @_;
4054
4055     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4056     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4057         if ! -e $source;
4058
4059     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4060     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4061         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4062         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4063         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4064             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4065             ." to the host.\n";
4066     }
4067 }
4068
4069 sub spice_port {
4070     my ($vmid) = @_;
4071
4072     my $res = mon_cmd($vmid, 'query-spice');
4073
4074     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4075 }
4076
4077 sub vm_devices_list {
4078     my ($vmid) = @_;
4079
4080     my $res = mon_cmd($vmid, 'query-pci');
4081     my $devices_to_check = [];
4082     my $devices = {};
4083     foreach my $pcibus (@$res) {
4084         push @$devices_to_check, @{$pcibus->{devices}},
4085     }
4086
4087     while (@$devices_to_check) {
4088         my $to_check = [];
4089         for my $d (@$devices_to_check) {
4090             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4091             next if !$d->{'pci_bridge'};
4092
4093             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4094             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4095         }
4096         $devices_to_check = $to_check;
4097     }
4098
4099     my $resblock = mon_cmd($vmid, 'query-block');
4100     foreach my $block (@$resblock) {
4101         if($block->{device} =~ m/^drive-(\S+)/){
4102                 $devices->{$1} = 1;
4103         }
4104     }
4105
4106     my $resmice = mon_cmd($vmid, 'query-mice');
4107     foreach my $mice (@$resmice) {
4108         if ($mice->{name} eq 'QEMU HID Tablet') {
4109             $devices->{tablet} = 1;
4110             last;
4111         }
4112     }
4113
4114     # for usb devices there is no query-usb
4115     # but we can iterate over the entries in
4116     # qom-list path=/machine/peripheral
4117     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4118     foreach my $per (@$resperipheral) {
4119         if ($per->{name} =~ m/^usb\d+$/) {
4120             $devices->{$per->{name}} = 1;
4121         }
4122     }
4123
4124     return $devices;
4125 }
4126
4127 sub vm_deviceplug {
4128     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4129
4130     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4131
4132     my $devices_list = vm_devices_list($vmid);
4133     return 1 if defined($devices_list->{$deviceid});
4134
4135     # add PCI bridge if we need it for the device
4136     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4137
4138     if ($deviceid eq 'tablet') {
4139         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4140     } elsif ($deviceid eq 'keyboard') {
4141         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4142     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4143         die "usb hotplug currently not reliable\n";
4144         # since we can't reliably hot unplug all added usb devices and usb
4145         # passthrough breaks live migration we disable usb hotplugging for now
4146         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4147     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4148         qemu_iothread_add($vmid, $deviceid, $device);
4149
4150         qemu_driveadd($storecfg, $vmid, $device);
4151         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4152
4153         qemu_deviceadd($vmid, $devicefull);
4154         eval { qemu_deviceaddverify($vmid, $deviceid); };
4155         if (my $err = $@) {
4156             eval { qemu_drivedel($vmid, $deviceid); };
4157             warn $@ if $@;
4158             die $err;
4159         }
4160     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4161         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4162         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4163         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4164
4165         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4166
4167         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4168             qemu_iothread_add($vmid, $deviceid, $device);
4169             $devicefull .= ",iothread=iothread-$deviceid";
4170         }
4171
4172         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4173             $devicefull .= ",num_queues=$device->{queues}";
4174         }
4175
4176         qemu_deviceadd($vmid, $devicefull);
4177         qemu_deviceaddverify($vmid, $deviceid);
4178     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4179         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4180         qemu_driveadd($storecfg, $vmid, $device);
4181
4182         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4183         eval { qemu_deviceadd($vmid, $devicefull); };
4184         if (my $err = $@) {
4185             eval { qemu_drivedel($vmid, $deviceid); };
4186             warn $@ if $@;
4187             die $err;
4188         }
4189     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4190         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4191
4192         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4193         my $use_old_bios_files = undef;
4194         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4195
4196         my $netdevicefull = print_netdevice_full(
4197             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4198         qemu_deviceadd($vmid, $netdevicefull);
4199         eval {
4200             qemu_deviceaddverify($vmid, $deviceid);
4201             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4202         };
4203         if (my $err = $@) {
4204             eval { qemu_netdevdel($vmid, $deviceid); };
4205             warn $@ if $@;
4206             die $err;
4207         }
4208     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4209         my $bridgeid = $2;
4210         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4211         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4212
4213         qemu_deviceadd($vmid, $devicefull);
4214         qemu_deviceaddverify($vmid, $deviceid);
4215     } else {
4216         die "can't hotplug device '$deviceid'\n";
4217     }
4218
4219     return 1;
4220 }
4221
4222 # fixme: this should raise exceptions on error!
4223 sub vm_deviceunplug {
4224     my ($vmid, $conf, $deviceid) = @_;
4225
4226     my $devices_list = vm_devices_list($vmid);
4227     return 1 if !defined($devices_list->{$deviceid});
4228
4229     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4230     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4231
4232     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4233         qemu_devicedel($vmid, $deviceid);
4234     } elsif ($deviceid =~ m/^usb\d+$/) {
4235         die "usb hotplug currently not reliable\n";
4236         # when unplugging usb devices this way, there may be remaining usb
4237         # controllers/hubs so we disable it for now
4238         #qemu_devicedel($vmid, $deviceid);
4239         #qemu_devicedelverify($vmid, $deviceid);
4240     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4241         my $device = parse_drive($deviceid, $conf->{$deviceid});
4242
4243         qemu_devicedel($vmid, $deviceid);
4244         qemu_devicedelverify($vmid, $deviceid);
4245         qemu_drivedel($vmid, $deviceid);
4246         qemu_iothread_del($vmid, $deviceid, $device);
4247     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4248         qemu_devicedel($vmid, $deviceid);
4249         qemu_devicedelverify($vmid, $deviceid);
4250     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4251         my $device = parse_drive($deviceid, $conf->{$deviceid});
4252
4253         qemu_devicedel($vmid, $deviceid);
4254         qemu_drivedel($vmid, $deviceid);
4255         qemu_deletescsihw($conf, $vmid, $deviceid);
4256
4257         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4258             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4259     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4260         qemu_devicedel($vmid, $deviceid);
4261         qemu_devicedelverify($vmid, $deviceid);
4262         qemu_netdevdel($vmid, $deviceid);
4263     } else {
4264         die "can't unplug device '$deviceid'\n";
4265     }
4266
4267     return 1;
4268 }
4269
4270 sub qemu_deviceadd {
4271     my ($vmid, $devicefull) = @_;
4272
4273     $devicefull = "driver=".$devicefull;
4274     my %options =  split(/[=,]/, $devicefull);
4275
4276     mon_cmd($vmid, "device_add" , %options);
4277 }
4278
4279 sub qemu_devicedel {
4280     my ($vmid, $deviceid) = @_;
4281
4282     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4283 }
4284
4285 sub qemu_iothread_add {
4286     my ($vmid, $deviceid, $device) = @_;
4287
4288     if ($device->{iothread}) {
4289         my $iothreads = vm_iothreads_list($vmid);
4290         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4291     }
4292 }
4293
4294 sub qemu_iothread_del {
4295     my ($vmid, $deviceid, $device) = @_;
4296
4297     if ($device->{iothread}) {
4298         my $iothreads = vm_iothreads_list($vmid);
4299         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4300     }
4301 }
4302
4303 sub qemu_objectadd {
4304     my ($vmid, $objectid, $qomtype) = @_;
4305
4306     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4307
4308     return 1;
4309 }
4310
4311 sub qemu_objectdel {
4312     my ($vmid, $objectid) = @_;
4313
4314     mon_cmd($vmid, "object-del", id => $objectid);
4315
4316     return 1;
4317 }
4318
4319 sub qemu_driveadd {
4320     my ($storecfg, $vmid, $device) = @_;
4321
4322     my $kvmver = get_running_qemu_version($vmid);
4323     my $io_uring = min_version($kvmver, 6, 0);
4324     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4325     $drive =~ s/\\/\\\\/g;
4326     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4327
4328     # If the command succeeds qemu prints: "OK"
4329     return 1 if $ret =~ m/OK/s;
4330
4331     die "adding drive failed: $ret\n";
4332 }
4333
4334 sub qemu_drivedel {
4335     my ($vmid, $deviceid) = @_;
4336
4337     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4338     $ret =~ s/^\s+//;
4339
4340     return 1 if $ret eq "";
4341
4342     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4343     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4344
4345     die "deleting drive $deviceid failed : $ret\n";
4346 }
4347
4348 sub qemu_deviceaddverify {
4349     my ($vmid, $deviceid) = @_;
4350
4351     for (my $i = 0; $i <= 5; $i++) {
4352          my $devices_list = vm_devices_list($vmid);
4353          return 1 if defined($devices_list->{$deviceid});
4354          sleep 1;
4355     }
4356
4357     die "error on hotplug device '$deviceid'\n";
4358 }
4359
4360
4361 sub qemu_devicedelverify {
4362     my ($vmid, $deviceid) = @_;
4363
4364     # need to verify that the device is correctly removed as device_del
4365     # is async and empty return is not reliable
4366
4367     for (my $i = 0; $i <= 5; $i++) {
4368          my $devices_list = vm_devices_list($vmid);
4369          return 1 if !defined($devices_list->{$deviceid});
4370          sleep 1;
4371     }
4372
4373     die "error on hot-unplugging device '$deviceid'\n";
4374 }
4375
4376 sub qemu_findorcreatescsihw {
4377     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4378
4379     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4380
4381     my $scsihwid="$controller_prefix$controller";
4382     my $devices_list = vm_devices_list($vmid);
4383
4384     if (!defined($devices_list->{$scsihwid})) {
4385         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4386     }
4387
4388     return 1;
4389 }
4390
4391 sub qemu_deletescsihw {
4392     my ($conf, $vmid, $opt) = @_;
4393
4394     my $device = parse_drive($opt, $conf->{$opt});
4395
4396     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4397         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4398         return 1;
4399     }
4400
4401     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4402
4403     my $devices_list = vm_devices_list($vmid);
4404     foreach my $opt (keys %{$devices_list}) {
4405         if (is_valid_drivename($opt)) {
4406             my $drive = parse_drive($opt, $conf->{$opt});
4407             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4408                 return 1;
4409             }
4410         }
4411     }
4412
4413     my $scsihwid="scsihw$controller";
4414
4415     vm_deviceunplug($vmid, $conf, $scsihwid);
4416
4417     return 1;
4418 }
4419
4420 sub qemu_add_pci_bridge {
4421     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4422
4423     my $bridges = {};
4424
4425     my $bridgeid;
4426
4427     print_pci_addr($device, $bridges, $arch, $machine_type);
4428
4429     while (my ($k, $v) = each %$bridges) {
4430         $bridgeid = $k;
4431     }
4432     return 1 if !defined($bridgeid) || $bridgeid < 1;
4433
4434     my $bridge = "pci.$bridgeid";
4435     my $devices_list = vm_devices_list($vmid);
4436
4437     if (!defined($devices_list->{$bridge})) {
4438         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4439     }
4440
4441     return 1;
4442 }
4443
4444 sub qemu_set_link_status {
4445     my ($vmid, $device, $up) = @_;
4446
4447     mon_cmd($vmid, "set_link", name => $device,
4448                up => $up ? JSON::true : JSON::false);
4449 }
4450
4451 sub qemu_netdevadd {
4452     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4453
4454     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4455     my %options =  split(/[=,]/, $netdev);
4456
4457     if (defined(my $vhost = $options{vhost})) {
4458         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4459     }
4460
4461     if (defined(my $queues = $options{queues})) {
4462         $options{queues} = $queues + 0;
4463     }
4464
4465     mon_cmd($vmid, "netdev_add",  %options);
4466     return 1;
4467 }
4468
4469 sub qemu_netdevdel {
4470     my ($vmid, $deviceid) = @_;
4471
4472     mon_cmd($vmid, "netdev_del", id => $deviceid);
4473 }
4474
4475 sub qemu_usb_hotplug {
4476     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4477
4478     return if !$device;
4479
4480     # remove the old one first
4481     vm_deviceunplug($vmid, $conf, $deviceid);
4482
4483     # check if xhci controller is necessary and available
4484     if ($device->{usb3}) {
4485
4486         my $devicelist = vm_devices_list($vmid);
4487
4488         if (!$devicelist->{xhci}) {
4489             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4490             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4491         }
4492     }
4493     my $d = parse_usb_device($device->{host});
4494     $d->{usb3} = $device->{usb3};
4495
4496     # add the new one
4497     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4498 }
4499
4500 sub qemu_cpu_hotplug {
4501     my ($vmid, $conf, $vcpus) = @_;
4502
4503     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4504
4505     my $sockets = 1;
4506     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4507     $sockets = $conf->{sockets} if  $conf->{sockets};
4508     my $cores = $conf->{cores} || 1;
4509     my $maxcpus = $sockets * $cores;
4510
4511     $vcpus = $maxcpus if !$vcpus;
4512
4513     die "you can't add more vcpus than maxcpus\n"
4514         if $vcpus > $maxcpus;
4515
4516     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4517
4518     if ($vcpus < $currentvcpus) {
4519
4520         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4521
4522             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4523                 qemu_devicedel($vmid, "cpu$i");
4524                 my $retry = 0;
4525                 my $currentrunningvcpus = undef;
4526                 while (1) {
4527                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4528                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4529                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4530                     $retry++;
4531                     sleep 1;
4532                 }
4533                 #update conf after each succesfull cpu unplug
4534                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4535                 PVE::QemuConfig->write_config($vmid, $conf);
4536             }
4537         } else {
4538             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4539         }
4540
4541         return;
4542     }
4543
4544     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4545     die "vcpus in running vm does not match its configuration\n"
4546         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4547
4548     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4549
4550         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4551             my $cpustr = print_cpu_device($conf, $i);
4552             qemu_deviceadd($vmid, $cpustr);
4553
4554             my $retry = 0;
4555             my $currentrunningvcpus = undef;
4556             while (1) {
4557                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4558                 last if scalar(@{$currentrunningvcpus}) == $i;
4559                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4560                 sleep 1;
4561                 $retry++;
4562             }
4563             #update conf after each succesfull cpu hotplug
4564             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4565             PVE::QemuConfig->write_config($vmid, $conf);
4566         }
4567     } else {
4568
4569         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4570             mon_cmd($vmid, "cpu-add", id => int($i));
4571         }
4572     }
4573 }
4574
4575 sub qemu_block_set_io_throttle {
4576     my ($vmid, $deviceid,
4577         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4578         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4579         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4580         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4581
4582     return if !check_running($vmid) ;
4583
4584     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4585         bps => int($bps),
4586         bps_rd => int($bps_rd),
4587         bps_wr => int($bps_wr),
4588         iops => int($iops),
4589         iops_rd => int($iops_rd),
4590         iops_wr => int($iops_wr),
4591         bps_max => int($bps_max),
4592         bps_rd_max => int($bps_rd_max),
4593         bps_wr_max => int($bps_wr_max),
4594         iops_max => int($iops_max),
4595         iops_rd_max => int($iops_rd_max),
4596         iops_wr_max => int($iops_wr_max),
4597         bps_max_length => int($bps_max_length),
4598         bps_rd_max_length => int($bps_rd_max_length),
4599         bps_wr_max_length => int($bps_wr_max_length),
4600         iops_max_length => int($iops_max_length),
4601         iops_rd_max_length => int($iops_rd_max_length),
4602         iops_wr_max_length => int($iops_wr_max_length),
4603     );
4604
4605 }
4606
4607 sub qemu_block_resize {
4608     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4609
4610     my $running = check_running($vmid);
4611
4612     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4613
4614     return if !$running;
4615
4616     my $padding = (1024 - $size % 1024) % 1024;
4617     $size = $size + $padding;
4618
4619     mon_cmd(
4620         $vmid,
4621         "block_resize",
4622         device => $deviceid,
4623         size => int($size),
4624         timeout => 60,
4625     );
4626 }
4627
4628 sub qemu_volume_snapshot {
4629     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4630
4631     my $running = check_running($vmid);
4632
4633     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4634         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4635     } else {
4636         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4637     }
4638 }
4639
4640 sub qemu_volume_snapshot_delete {
4641     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4642
4643     my $running = check_running($vmid);
4644
4645     if($running) {
4646
4647         $running = undef;
4648         my $conf = PVE::QemuConfig->load_config($vmid);
4649         PVE::QemuConfig->foreach_volume($conf, sub {
4650             my ($ds, $drive) = @_;
4651             $running = 1 if $drive->{file} eq $volid;
4652         });
4653     }
4654
4655     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4656         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4657     } else {
4658         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4659     }
4660 }
4661
4662 sub set_migration_caps {
4663     my ($vmid, $savevm) = @_;
4664
4665     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4666
4667     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4668     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4669
4670     my $cap_ref = [];
4671
4672     my $enabled_cap = {
4673         "auto-converge" => 1,
4674         "xbzrle" => 1,
4675         "x-rdma-pin-all" => 0,
4676         "zero-blocks" => 0,
4677         "compress" => 0,
4678         "dirty-bitmaps" => $dirty_bitmaps,
4679     };
4680
4681     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4682
4683     for my $supported_capability (@$supported_capabilities) {
4684         push @$cap_ref, {
4685             capability => $supported_capability->{capability},
4686             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4687         };
4688     }
4689
4690     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4691 }
4692
4693 sub foreach_volid {
4694     my ($conf, $func, @param) = @_;
4695
4696     my $volhash = {};
4697
4698     my $test_volid = sub {
4699         my ($key, $drive, $snapname) = @_;
4700
4701         my $volid = $drive->{file};
4702         return if !$volid;
4703
4704         $volhash->{$volid}->{cdrom} //= 1;
4705         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4706
4707         my $replicate = $drive->{replicate} // 1;
4708         $volhash->{$volid}->{replicate} //= 0;
4709         $volhash->{$volid}->{replicate} = 1 if $replicate;
4710
4711         $volhash->{$volid}->{shared} //= 0;
4712         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4713
4714         $volhash->{$volid}->{referenced_in_config} //= 0;
4715         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4716
4717         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4718             if defined($snapname);
4719
4720         my $size = $drive->{size};
4721         $volhash->{$volid}->{size} //= $size if $size;
4722
4723         $volhash->{$volid}->{is_vmstate} //= 0;
4724         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4725
4726         $volhash->{$volid}->{is_tpmstate} //= 0;
4727         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4728
4729         $volhash->{$volid}->{is_unused} //= 0;
4730         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4731
4732         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4733     };
4734
4735     my $include_opts = {
4736         extra_keys => ['vmstate'],
4737         include_unused => 1,
4738     };
4739
4740     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4741     foreach my $snapname (keys %{$conf->{snapshots}}) {
4742         my $snap = $conf->{snapshots}->{$snapname};
4743         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4744     }
4745
4746     foreach my $volid (keys %$volhash) {
4747         &$func($volid, $volhash->{$volid}, @param);
4748     }
4749 }
4750
4751 my $fast_plug_option = {
4752     'lock' => 1,
4753     'name' => 1,
4754     'onboot' => 1,
4755     'shares' => 1,
4756     'startup' => 1,
4757     'description' => 1,
4758     'protection' => 1,
4759     'vmstatestorage' => 1,
4760     'hookscript' => 1,
4761     'tags' => 1,
4762 };
4763
4764 # hotplug changes in [PENDING]
4765 # $selection hash can be used to only apply specified options, for
4766 # example: { cores => 1 } (only apply changed 'cores')
4767 # $errors ref is used to return error messages
4768 sub vmconfig_hotplug_pending {
4769     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4770
4771     my $defaults = load_defaults();
4772     my $arch = get_vm_arch($conf);
4773     my $machine_type = get_vm_machine($conf, undef, $arch);
4774
4775     # commit values which do not have any impact on running VM first
4776     # Note: those option cannot raise errors, we we do not care about
4777     # $selection and always apply them.
4778
4779     my $add_error = sub {
4780         my ($opt, $msg) = @_;
4781         $errors->{$opt} = "hotplug problem - $msg";
4782     };
4783
4784     my $changes = 0;
4785     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4786         if ($fast_plug_option->{$opt}) {
4787             $conf->{$opt} = $conf->{pending}->{$opt};
4788             delete $conf->{pending}->{$opt};
4789             $changes = 1;
4790         }
4791     }
4792
4793     if ($changes) {
4794         PVE::QemuConfig->write_config($vmid, $conf);
4795     }
4796
4797     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4798
4799     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4800     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4801     foreach my $opt (sort keys %$pending_delete_hash) {
4802         next if $selection && !$selection->{$opt};
4803         my $force = $pending_delete_hash->{$opt}->{force};
4804         eval {
4805             if ($opt eq 'hotplug') {
4806                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4807             } elsif ($opt eq 'tablet') {
4808                 die "skip\n" if !$hotplug_features->{usb};
4809                 if ($defaults->{tablet}) {
4810                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4811                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4812                         if $arch eq 'aarch64';
4813                 } else {
4814                     vm_deviceunplug($vmid, $conf, 'tablet');
4815                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4816                 }
4817             } elsif ($opt =~ m/^usb\d+/) {
4818                 die "skip\n";
4819                 # since we cannot reliably hot unplug usb devices we are disabling it
4820                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4821                 #vm_deviceunplug($vmid, $conf, $opt);
4822             } elsif ($opt eq 'vcpus') {
4823                 die "skip\n" if !$hotplug_features->{cpu};
4824                 qemu_cpu_hotplug($vmid, $conf, undef);
4825             } elsif ($opt eq 'balloon') {
4826                 # enable balloon device is not hotpluggable
4827                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4828                 # here we reset the ballooning value to memory
4829                 my $balloon = $conf->{memory} || $defaults->{memory};
4830                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4831             } elsif ($fast_plug_option->{$opt}) {
4832                 # do nothing
4833             } elsif ($opt =~ m/^net(\d+)$/) {
4834                 die "skip\n" if !$hotplug_features->{network};
4835                 vm_deviceunplug($vmid, $conf, $opt);
4836             } elsif (is_valid_drivename($opt)) {
4837                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4838                 vm_deviceunplug($vmid, $conf, $opt);
4839                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4840             } elsif ($opt =~ m/^memory$/) {
4841                 die "skip\n" if !$hotplug_features->{memory};
4842                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4843             } elsif ($opt eq 'cpuunits') {
4844                 $cgroup->change_cpu_shares(undef, 1024);
4845             } elsif ($opt eq 'cpulimit') {
4846                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4847             } else {
4848                 die "skip\n";
4849             }
4850         };
4851         if (my $err = $@) {
4852             &$add_error($opt, $err) if $err ne "skip\n";
4853         } else {
4854             delete $conf->{$opt};
4855             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4856         }
4857     }
4858
4859     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4860     $apply_pending_cloudinit = sub {
4861         return if $apply_pending_cloudinit_done; # once is enough
4862         $apply_pending_cloudinit_done = 1; # once is enough
4863
4864         my ($key, $value) = @_;
4865
4866         my @cloudinit_opts = keys %$confdesc_cloudinit;
4867         foreach my $opt (keys %{$conf->{pending}}) {
4868             next if !grep { $_ eq $opt } @cloudinit_opts;
4869             $conf->{$opt} = delete $conf->{pending}->{$opt};
4870         }
4871
4872         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4873         foreach my $opt (sort keys %$pending_delete_hash) {
4874             next if !grep { $_ eq $opt } @cloudinit_opts;
4875             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4876             delete $conf->{$opt};
4877         }
4878
4879         my $new_conf = { %$conf };
4880         $new_conf->{$key} = $value;
4881         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4882     };
4883
4884     foreach my $opt (keys %{$conf->{pending}}) {
4885         next if $selection && !$selection->{$opt};
4886         my $value = $conf->{pending}->{$opt};
4887         eval {
4888             if ($opt eq 'hotplug') {
4889                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4890             } elsif ($opt eq 'tablet') {
4891                 die "skip\n" if !$hotplug_features->{usb};
4892                 if ($value == 1) {
4893                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4894                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4895                         if $arch eq 'aarch64';
4896                 } elsif ($value == 0) {
4897                     vm_deviceunplug($vmid, $conf, 'tablet');
4898                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4899                 }
4900             } elsif ($opt =~ m/^usb\d+$/) {
4901                 die "skip\n";
4902                 # since we cannot reliably hot unplug usb devices we disable it for now
4903                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4904                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4905                 #die "skip\n" if !$d;
4906                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4907             } elsif ($opt eq 'vcpus') {
4908                 die "skip\n" if !$hotplug_features->{cpu};
4909                 qemu_cpu_hotplug($vmid, $conf, $value);
4910             } elsif ($opt eq 'balloon') {
4911                 # enable/disable balloning device is not hotpluggable
4912                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4913                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4914                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4915
4916                 # allow manual ballooning if shares is set to zero
4917                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4918                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4919                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4920                 }
4921             } elsif ($opt =~ m/^net(\d+)$/) {
4922                 # some changes can be done without hotplug
4923                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4924                                     $vmid, $opt, $value, $arch, $machine_type);
4925             } elsif (is_valid_drivename($opt)) {
4926                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4927                 # some changes can be done without hotplug
4928                 my $drive = parse_drive($opt, $value);
4929                 if (drive_is_cloudinit($drive)) {
4930                     &$apply_pending_cloudinit($opt, $value);
4931                 }
4932                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4933                                      $vmid, $opt, $value, $arch, $machine_type);
4934             } elsif ($opt =~ m/^memory$/) { #dimms
4935                 die "skip\n" if !$hotplug_features->{memory};
4936                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4937             } elsif ($opt eq 'cpuunits') {
4938                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4939             } elsif ($opt eq 'cpulimit') {
4940                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4941                 $cgroup->change_cpu_quota($cpulimit, 100000);
4942             } elsif ($opt eq 'agent') {
4943                 vmconfig_update_agent($conf, $opt, $value);
4944             } else {
4945                 die "skip\n";  # skip non-hot-pluggable options
4946             }
4947         };
4948         if (my $err = $@) {
4949             &$add_error($opt, $err) if $err ne "skip\n";
4950         } else {
4951             $conf->{$opt} = $value;
4952             delete $conf->{pending}->{$opt};
4953         }
4954     }
4955
4956     PVE::QemuConfig->write_config($vmid, $conf);
4957 }
4958
4959 sub try_deallocate_drive {
4960     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4961
4962     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4963         my $volid = $drive->{file};
4964         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4965             my $sid = PVE::Storage::parse_volume_id($volid);
4966             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4967
4968             # check if the disk is really unused
4969             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4970                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4971             PVE::Storage::vdisk_free($storecfg, $volid);
4972             return 1;
4973         } else {
4974             # If vm is not owner of this disk remove from config
4975             return 1;
4976         }
4977     }
4978
4979     return;
4980 }
4981
4982 sub vmconfig_delete_or_detach_drive {
4983     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4984
4985     my $drive = parse_drive($opt, $conf->{$opt});
4986
4987     my $rpcenv = PVE::RPCEnvironment::get();
4988     my $authuser = $rpcenv->get_user();
4989
4990     if ($force) {
4991         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4992         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4993     } else {
4994         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4995     }
4996 }
4997
4998
4999
5000 sub vmconfig_apply_pending {
5001     my ($vmid, $conf, $storecfg, $errors) = @_;
5002
5003     return if !scalar(keys %{$conf->{pending}});
5004
5005     my $add_apply_error = sub {
5006         my ($opt, $msg) = @_;
5007         my $err_msg = "unable to apply pending change $opt : $msg";
5008         $errors->{$opt} = $err_msg;
5009         warn $err_msg;
5010     };
5011
5012     # cold plug
5013
5014     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5015     foreach my $opt (sort keys %$pending_delete_hash) {
5016         my $force = $pending_delete_hash->{$opt}->{force};
5017         eval {
5018             if ($opt =~ m/^unused/) {
5019                 die "internal error";
5020             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5021                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5022             }
5023         };
5024         if (my $err = $@) {
5025             $add_apply_error->($opt, $err);
5026         } else {
5027             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5028             delete $conf->{$opt};
5029         }
5030     }
5031
5032     PVE::QemuConfig->cleanup_pending($conf);
5033
5034     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5035         next if $opt eq 'delete'; # just to be sure
5036         eval {
5037             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5038                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5039             }
5040         };
5041         if (my $err = $@) {
5042             $add_apply_error->($opt, $err);
5043         } else {
5044             $conf->{$opt} = delete $conf->{pending}->{$opt};
5045         }
5046     }
5047
5048     # write all changes at once to avoid unnecessary i/o
5049     PVE::QemuConfig->write_config($vmid, $conf);
5050 }
5051
5052 sub vmconfig_update_net {
5053     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5054
5055     my $newnet = parse_net($value);
5056
5057     if ($conf->{$opt}) {
5058         my $oldnet = parse_net($conf->{$opt});
5059
5060         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5061             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5062             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5063             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5064
5065             # for non online change, we try to hot-unplug
5066             die "skip\n" if !$hotplug;
5067             vm_deviceunplug($vmid, $conf, $opt);
5068         } else {
5069
5070             die "internal error" if $opt !~ m/net(\d+)/;
5071             my $iface = "tap${vmid}i$1";
5072
5073             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5074                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5075                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5076                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5077                 PVE::Network::tap_unplug($iface);
5078
5079                 if ($have_sdn) {
5080                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5081                 } else {
5082                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5083                 }
5084             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5085                 # Rate can be applied on its own but any change above needs to
5086                 # include the rate in tap_plug since OVS resets everything.
5087                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5088             }
5089
5090             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5091                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5092             }
5093
5094             return 1;
5095         }
5096     }
5097
5098     if ($hotplug) {
5099         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5100     } else {
5101         die "skip\n";
5102     }
5103 }
5104
5105 sub vmconfig_update_agent {
5106     my ($conf, $opt, $value) = @_;
5107
5108     die "skip\n" if !$conf->{$opt};
5109
5110     my $hotplug_options = { fstrim_cloned_disks => 1 };
5111
5112     my $old_agent = parse_guest_agent($conf);
5113     my $agent = parse_guest_agent({$opt => $value});
5114
5115     for my $option (keys %$agent) { # added/changed options
5116         next if defined($hotplug_options->{$option});
5117         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5118     }
5119
5120     for my $option (keys %$old_agent) { # removed options
5121         next if defined($hotplug_options->{$option});
5122         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5123     }
5124
5125     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5126 }
5127
5128 sub vmconfig_update_disk {
5129     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5130
5131     my $drive = parse_drive($opt, $value);
5132
5133     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5134         my $media = $drive->{media} || 'disk';
5135         my $oldmedia = $old_drive->{media} || 'disk';
5136         die "unable to change media type\n" if $media ne $oldmedia;
5137
5138         if (!drive_is_cdrom($old_drive)) {
5139
5140             if ($drive->{file} ne $old_drive->{file}) {
5141
5142                 die "skip\n" if !$hotplug;
5143
5144                 # unplug and register as unused
5145                 vm_deviceunplug($vmid, $conf, $opt);
5146                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5147
5148             } else {
5149                 # update existing disk
5150
5151                 # skip non hotpluggable value
5152                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5153                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5154                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5155                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5156                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5157                     die "skip\n";
5158                 }
5159
5160                 # apply throttle
5161                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5162                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5163                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5164                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5165                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5166                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5167                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5168                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5169                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5170                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5171                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5172                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5173                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5174                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5175                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5176                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5177                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5178                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5179
5180                     qemu_block_set_io_throttle(
5181                         $vmid,"drive-$opt",
5182                         ($drive->{mbps} || 0)*1024*1024,
5183                         ($drive->{mbps_rd} || 0)*1024*1024,
5184                         ($drive->{mbps_wr} || 0)*1024*1024,
5185                          $drive->{iops} || 0,
5186                          $drive->{iops_rd} || 0,
5187                          $drive->{iops_wr} || 0,
5188                         ($drive->{mbps_max} || 0)*1024*1024,
5189                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5190                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5191                          $drive->{iops_max} || 0,
5192                          $drive->{iops_rd_max} || 0,
5193                          $drive->{iops_wr_max} || 0,
5194                          $drive->{bps_max_length} || 1,
5195                          $drive->{bps_rd_max_length} || 1,
5196                          $drive->{bps_wr_max_length} || 1,
5197                          $drive->{iops_max_length} || 1,
5198                          $drive->{iops_rd_max_length} || 1,
5199                          $drive->{iops_wr_max_length} || 1,
5200                     );
5201
5202                 }
5203
5204                 return 1;
5205             }
5206
5207         } else { # cdrom
5208
5209             if ($drive->{file} eq 'none') {
5210                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5211                 if (drive_is_cloudinit($old_drive)) {
5212                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5213                 }
5214             } else {
5215                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5216
5217                 # force eject if locked
5218                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5219
5220                 if ($path) {
5221                     mon_cmd($vmid, "blockdev-change-medium",
5222                         id => "$opt", filename => "$path");
5223                 }
5224             }
5225
5226             return 1;
5227         }
5228     }
5229
5230     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5231     # hotplug new disks
5232     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5233     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5234 }
5235
5236 # called in locked context by incoming migration
5237 sub vm_migrate_get_nbd_disks {
5238     my ($storecfg, $conf, $replicated_volumes) = @_;
5239
5240     my $local_volumes = {};
5241     PVE::QemuConfig->foreach_volume($conf, sub {
5242         my ($ds, $drive) = @_;
5243
5244         return if drive_is_cdrom($drive);
5245         return if $ds eq 'tpmstate0';
5246
5247         my $volid = $drive->{file};
5248
5249         return if !$volid;
5250
5251         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5252
5253         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5254         return if $scfg->{shared};
5255
5256         # replicated disks re-use existing state via bitmap
5257         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5258         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5259     });
5260     return $local_volumes;
5261 }
5262
5263 # called in locked context by incoming migration
5264 sub vm_migrate_alloc_nbd_disks {
5265     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5266
5267     my $nbd = {};
5268     foreach my $opt (sort keys %$source_volumes) {
5269         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5270
5271         if ($use_existing) {
5272             $nbd->{$opt}->{drivestr} = print_drive($drive);
5273             $nbd->{$opt}->{volid} = $volid;
5274             $nbd->{$opt}->{replicated} = 1;
5275             next;
5276         }
5277
5278         # storage mapping + volname = regular migration
5279         # storage mapping + format = remote migration
5280         # order of precedence, filtered by whether storage supports it:
5281         # 1. explicit requested format
5282         # 2. format of current volume
5283         # 3. default format of storage
5284         if (!$storagemap->{identity}) {
5285             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5286             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5287             if (!$format || !grep { $format eq $_ } @$validFormats) {
5288                 if ($volname) {
5289                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5290                     my $fileFormat = qemu_img_format($scfg, $volname);
5291                     $format = $fileFormat
5292                         if grep { $fileFormat eq $_ } @$validFormats;
5293                 }
5294                 $format //= $defFormat;
5295             }
5296         } else {
5297             # can't happen for remote migration, so $volname is always defined
5298             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5299             $format = qemu_img_format($scfg, $volname);
5300         }
5301
5302         my $size = $drive->{size} / 1024;
5303         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5304         my $newdrive = $drive;
5305         $newdrive->{format} = $format;
5306         $newdrive->{file} = $newvolid;
5307         my $drivestr = print_drive($newdrive);
5308         $nbd->{$opt}->{drivestr} = $drivestr;
5309         $nbd->{$opt}->{volid} = $newvolid;
5310     }
5311
5312     return $nbd;
5313 }
5314
5315 # see vm_start_nolock for parameters, additionally:
5316 # migrate_opts:
5317 #   storagemap = parsed storage map for allocating NBD disks
5318 sub vm_start {
5319     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5320
5321     return PVE::QemuConfig->lock_config($vmid, sub {
5322         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5323
5324         die "you can't start a vm if it's a template\n"
5325             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5326
5327         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5328         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5329
5330         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5331
5332         if ($has_backup_lock && $running) {
5333             # a backup is currently running, attempt to start the guest in the
5334             # existing QEMU instance
5335             return vm_resume($vmid);
5336         }
5337
5338         PVE::QemuConfig->check_lock($conf)
5339             if !($params->{skiplock} || $has_suspended_lock);
5340
5341         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5342
5343         die "VM $vmid already running\n" if $running;
5344
5345         if (my $storagemap = $migrate_opts->{storagemap}) {
5346             my $replicated = $migrate_opts->{replicated_volumes};
5347             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5348             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5349
5350             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5351                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5352             }
5353         }
5354
5355         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5356     });
5357 }
5358
5359
5360 # params:
5361 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5362 #   skiplock => 0/1, skip checking for config lock
5363 #   skiptemplate => 0/1, skip checking whether VM is template
5364 #   forcemachine => to force Qemu machine (rollback/migration)
5365 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5366 #   timeout => in seconds
5367 #   paused => start VM in paused state (backup)
5368 #   resume => resume from hibernation
5369 #   pbs-backing => {
5370 #      sata0 => {
5371 #         repository
5372 #         snapshot
5373 #         keyfile
5374 #         archive
5375 #      },
5376 #      virtio2 => ...
5377 #   }
5378 # migrate_opts:
5379 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5380 #   migratedfrom => source node
5381 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5382 #   network => CIDR of migration network
5383 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5384 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5385 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5386 #   tpmstate_vol => new volid of tpmstate0, not yet contained in config
5387 sub vm_start_nolock {
5388     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5389
5390     my $statefile = $params->{statefile};
5391     my $resume = $params->{resume};
5392
5393     my $migratedfrom = $migrate_opts->{migratedfrom};
5394     my $migration_type = $migrate_opts->{type};
5395
5396     my $res = {};
5397
5398     # clean up leftover reboot request files
5399     eval { clear_reboot_request($vmid); };
5400     warn $@ if $@;
5401
5402     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5403         vmconfig_apply_pending($vmid, $conf, $storecfg);
5404         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5405     }
5406
5407     # don't regenerate the ISO if the VM is started as part of a live migration
5408     # this way we can reuse the old ISO with the correct config
5409     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5410
5411     # override TPM state vol if migrated, conf is out of date still
5412     if (my $tpmvol = $migrate_opts->{tpmstate_vol}) {
5413         my $parsed = parse_drive("tpmstate0", $conf->{tpmstate0});
5414         $parsed->{file} = $tpmvol;
5415         $conf->{tpmstate0} = print_drive($parsed);
5416     }
5417
5418     my $defaults = load_defaults();
5419
5420     # set environment variable useful inside network script
5421     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5422
5423     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5424
5425     my $forcemachine = $params->{forcemachine};
5426     my $forcecpu = $params->{forcecpu};
5427     if ($resume) {
5428         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5429         $forcemachine = $conf->{runningmachine};
5430         $forcecpu = $conf->{runningcpu};
5431         print "Resuming suspended VM\n";
5432     }
5433
5434     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5435         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5436
5437     my $migration_ip;
5438     my $get_migration_ip = sub {
5439         my ($nodename) = @_;
5440
5441         return $migration_ip if defined($migration_ip);
5442
5443         my $cidr = $migrate_opts->{network};
5444
5445         if (!defined($cidr)) {
5446             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5447             $cidr = $dc_conf->{migration}->{network};
5448         }
5449
5450         if (defined($cidr)) {
5451             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5452
5453             die "could not get IP: no address configured on local " .
5454                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5455
5456             die "could not get IP: multiple addresses configured on local " .
5457                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5458
5459             $migration_ip = @$ips[0];
5460         }
5461
5462         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5463             if !defined($migration_ip);
5464
5465         return $migration_ip;
5466     };
5467
5468     my $migrate_uri;
5469     if ($statefile) {
5470         if ($statefile eq 'tcp') {
5471             my $localip = "localhost";
5472             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5473             my $nodename = nodename();
5474
5475             if (!defined($migration_type)) {
5476                 if (defined($datacenterconf->{migration}->{type})) {
5477                     $migration_type = $datacenterconf->{migration}->{type};
5478                 } else {
5479                     $migration_type = 'secure';
5480                 }
5481             }
5482
5483             if ($migration_type eq 'insecure') {
5484                 $localip = $get_migration_ip->($nodename);
5485                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5486             }
5487
5488             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5489             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5490             $migrate_uri = "tcp:${localip}:${migrate_port}";
5491             push @$cmd, '-incoming', $migrate_uri;
5492             push @$cmd, '-S';
5493
5494         } elsif ($statefile eq 'unix') {
5495             # should be default for secure migrations as a ssh TCP forward
5496             # tunnel is not deterministic reliable ready and fails regurarly
5497             # to set up in time, so use UNIX socket forwards
5498             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5499             unlink $socket_addr;
5500
5501             $migrate_uri = "unix:$socket_addr";
5502
5503             push @$cmd, '-incoming', $migrate_uri;
5504             push @$cmd, '-S';
5505
5506         } elsif (-e $statefile) {
5507             push @$cmd, '-loadstate', $statefile;
5508         } else {
5509             my $statepath = PVE::Storage::path($storecfg, $statefile);
5510             push @$vollist, $statefile;
5511             push @$cmd, '-loadstate', $statepath;
5512         }
5513     } elsif ($params->{paused}) {
5514         push @$cmd, '-S';
5515     }
5516
5517     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5518
5519     my $pci_devices = {}; # host pci devices
5520     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5521         my $dev = $conf->{"hostpci$i"} or next;
5522         $pci_devices->{$i} = parse_hostpci($dev);
5523     }
5524
5525     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5526     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5527
5528     # map to a flat list of pci ids
5529     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5530
5531     # reserve all PCI IDs before actually doing anything with them
5532     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5533
5534     eval {
5535         for my $id (sort keys %$pci_devices) {
5536             my $d = $pci_devices->{$id};
5537             for my $dev ($d->{pciid}->@*) {
5538                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5539             }
5540         }
5541     };
5542     if (my $err = $@) {
5543         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5544         warn $@ if $@;
5545         die $err;
5546     }
5547
5548     PVE::Storage::activate_volumes($storecfg, $vollist);
5549
5550     eval {
5551         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5552     };
5553     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5554     # timeout should be more than enough here...
5555     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5556
5557     my $cpuunits = get_cpuunits($conf);
5558
5559     my %run_params = (
5560         timeout => $statefile ? undef : $start_timeout,
5561         umask => 0077,
5562         noerr => 1,
5563     );
5564
5565     # when migrating, prefix QEMU output so other side can pick up any
5566     # errors that might occur and show the user
5567     if ($migratedfrom) {
5568         $run_params{quiet} = 1;
5569         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5570     }
5571
5572     my %systemd_properties = (
5573         Slice => 'qemu.slice',
5574         KillMode => 'process',
5575         SendSIGKILL => 0,
5576         TimeoutStopUSec => ULONG_MAX, # infinity
5577     );
5578
5579     if (PVE::CGroup::cgroup_mode() == 2) {
5580         $systemd_properties{CPUWeight} = $cpuunits;
5581     } else {
5582         $systemd_properties{CPUShares} = $cpuunits;
5583     }
5584
5585     if (my $cpulimit = $conf->{cpulimit}) {
5586         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5587     }
5588     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5589
5590     my $run_qemu = sub {
5591         PVE::Tools::run_fork sub {
5592             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5593
5594             my $tpmpid;
5595             if (my $tpm = $conf->{tpmstate0}) {
5596                 # start the TPM emulator so QEMU can connect on start
5597                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5598             }
5599
5600             my $exitcode = run_command($cmd, %run_params);
5601             if ($exitcode) {
5602                 if ($tpmpid) {
5603                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5604                     kill 'TERM', $tpmpid;
5605                 }
5606                 die "QEMU exited with code $exitcode\n";
5607             }
5608         };
5609     };
5610
5611     if ($conf->{hugepages}) {
5612
5613         my $code = sub {
5614             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5615             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5616
5617             PVE::QemuServer::Memory::hugepages_mount();
5618             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5619
5620             eval { $run_qemu->() };
5621             if (my $err = $@) {
5622                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5623                     if !$conf->{keephugepages};
5624                 die $err;
5625             }
5626
5627             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5628                 if !$conf->{keephugepages};
5629         };
5630         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5631
5632     } else {
5633         eval { $run_qemu->() };
5634     }
5635
5636     if (my $err = $@) {
5637         # deactivate volumes if start fails
5638         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5639         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5640
5641         die "start failed: $err";
5642     }
5643
5644     # re-reserve all PCI IDs now that we can know the actual VM PID
5645     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5646     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5647     warn $@ if $@;
5648
5649     print "migration listens on $migrate_uri\n" if $migrate_uri;
5650     $res->{migrate_uri} = $migrate_uri;
5651
5652     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5653         eval { mon_cmd($vmid, "cont"); };
5654         warn $@ if $@;
5655     }
5656
5657     #start nbd server for storage migration
5658     if (my $nbd = $migrate_opts->{nbd}) {
5659         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5660
5661         my $migrate_storage_uri;
5662         # nbd_protocol_version > 0 for unix socket support
5663         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5664             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5665             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5666             $migrate_storage_uri = "nbd:unix:$socket_path";
5667         } else {
5668             my $nodename = nodename();
5669             my $localip = $get_migration_ip->($nodename);
5670             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5671             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5672
5673             mon_cmd($vmid, "nbd-server-start", addr => {
5674                 type => 'inet',
5675                 data => {
5676                     host => "${localip}",
5677                     port => "${storage_migrate_port}",
5678                 },
5679             });
5680             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5681             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5682         }
5683
5684         $res->{migrate_storage_uri} = $migrate_storage_uri;
5685
5686         foreach my $opt (sort keys %$nbd) {
5687             my $drivestr = $nbd->{$opt}->{drivestr};
5688             my $volid = $nbd->{$opt}->{volid};
5689             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5690             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5691             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5692             print "re-using replicated volume: $opt - $volid\n"
5693                 if $nbd->{$opt}->{replicated};
5694
5695             $res->{drives}->{$opt} = $nbd->{$opt};
5696             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5697         }
5698     }
5699
5700     if ($migratedfrom) {
5701         eval {
5702             set_migration_caps($vmid);
5703         };
5704         warn $@ if $@;
5705
5706         if ($spice_port) {
5707             print "spice listens on port $spice_port\n";
5708             $res->{spice_port} = $spice_port;
5709             if ($migrate_opts->{spice_ticket}) {
5710                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5711                     $migrate_opts->{spice_ticket});
5712                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5713             }
5714         }
5715
5716     } else {
5717         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5718             if !$statefile && $conf->{balloon};
5719
5720         foreach my $opt (keys %$conf) {
5721             next if $opt !~  m/^net\d+$/;
5722             my $nicconf = parse_net($conf->{$opt});
5723             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5724         }
5725     }
5726
5727     mon_cmd($vmid, 'qom-set',
5728                 path => "machine/peripheral/balloon0",
5729                 property => "guest-stats-polling-interval",
5730                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5731
5732     if ($resume) {
5733         print "Resumed VM, removing state\n";
5734         if (my $vmstate = $conf->{vmstate}) {
5735             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5736             PVE::Storage::vdisk_free($storecfg, $vmstate);
5737         }
5738         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5739         PVE::QemuConfig->write_config($vmid, $conf);
5740     }
5741
5742     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5743
5744     return $res;
5745 }
5746
5747 sub vm_commandline {
5748     my ($storecfg, $vmid, $snapname) = @_;
5749
5750     my $conf = PVE::QemuConfig->load_config($vmid);
5751
5752     my ($forcemachine, $forcecpu);
5753     if ($snapname) {
5754         my $snapshot = $conf->{snapshots}->{$snapname};
5755         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5756
5757         # check for machine or CPU overrides in snapshot
5758         $forcemachine = $snapshot->{runningmachine};
5759         $forcecpu = $snapshot->{runningcpu};
5760
5761         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5762
5763         $conf = $snapshot;
5764     }
5765
5766     my $defaults = load_defaults();
5767
5768     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5769
5770     return PVE::Tools::cmd2string($cmd);
5771 }
5772
5773 sub vm_reset {
5774     my ($vmid, $skiplock) = @_;
5775
5776     PVE::QemuConfig->lock_config($vmid, sub {
5777
5778         my $conf = PVE::QemuConfig->load_config($vmid);
5779
5780         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5781
5782         mon_cmd($vmid, "system_reset");
5783     });
5784 }
5785
5786 sub get_vm_volumes {
5787     my ($conf) = @_;
5788
5789     my $vollist = [];
5790     foreach_volid($conf, sub {
5791         my ($volid, $attr) = @_;
5792
5793         return if $volid =~ m|^/|;
5794
5795         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5796         return if !$sid;
5797
5798         push @$vollist, $volid;
5799     });
5800
5801     return $vollist;
5802 }
5803
5804 sub vm_stop_cleanup {
5805     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5806
5807     eval {
5808
5809         if (!$keepActive) {
5810             my $vollist = get_vm_volumes($conf);
5811             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5812
5813             if (my $tpmdrive = $conf->{tpmstate0}) {
5814                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5815                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5816                 if ($storeid) {
5817                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5818                 }
5819             }
5820         }
5821
5822         foreach my $ext (qw(mon qmp pid vnc qga)) {
5823             unlink "/var/run/qemu-server/${vmid}.$ext";
5824         }
5825
5826         if ($conf->{ivshmem}) {
5827             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5828             # just delete it for now, VMs which have this already open do not
5829             # are affected, but new VMs will get a separated one. If this
5830             # becomes an issue we either add some sort of ref-counting or just
5831             # add a "don't delete on stop" flag to the ivshmem format.
5832             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5833         }
5834
5835         my $ids = [];
5836         foreach my $key (keys %$conf) {
5837             next if $key !~ m/^hostpci(\d+)$/;
5838             my $hostpciindex = $1;
5839             my $d = parse_hostpci($conf->{$key});
5840             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5841
5842             foreach my $pci (@{$d->{pciid}}) {
5843                 my $pciid = $pci->{id};
5844                 push @$ids, $pci->{id};
5845                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5846             }
5847         }
5848         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5849
5850         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5851     };
5852     warn $@ if $@; # avoid errors - just warn
5853 }
5854
5855 # call only in locked context
5856 sub _do_vm_stop {
5857     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5858
5859     my $pid = check_running($vmid, $nocheck);
5860     return if !$pid;
5861
5862     my $conf;
5863     if (!$nocheck) {
5864         $conf = PVE::QemuConfig->load_config($vmid);
5865         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5866         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5867             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5868             $timeout = $opts->{down} if $opts->{down};
5869         }
5870         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5871     }
5872
5873     eval {
5874         if ($shutdown) {
5875             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5876                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5877             } else {
5878                 mon_cmd($vmid, "system_powerdown");
5879             }
5880         } else {
5881             mon_cmd($vmid, "quit");
5882         }
5883     };
5884     my $err = $@;
5885
5886     if (!$err) {
5887         $timeout = 60 if !defined($timeout);
5888
5889         my $count = 0;
5890         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5891             $count++;
5892             sleep 1;
5893         }
5894
5895         if ($count >= $timeout) {
5896             if ($force) {
5897                 warn "VM still running - terminating now with SIGTERM\n";
5898                 kill 15, $pid;
5899             } else {
5900                 die "VM quit/powerdown failed - got timeout\n";
5901             }
5902         } else {
5903             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5904             return;
5905         }
5906     } else {
5907        if (!check_running($vmid, $nocheck)) {
5908             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5909             return;
5910        }
5911        if ($force) {
5912             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5913             kill 15, $pid;
5914         } else {
5915             die "VM quit/powerdown failed\n";
5916         }
5917     }
5918
5919     # wait again
5920     $timeout = 10;
5921
5922     my $count = 0;
5923     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5924         $count++;
5925         sleep 1;
5926     }
5927
5928     if ($count >= $timeout) {
5929         warn "VM still running - terminating now with SIGKILL\n";
5930         kill 9, $pid;
5931         sleep 1;
5932     }
5933
5934     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5935 }
5936
5937 # Note: use $nocheck to skip tests if VM configuration file exists.
5938 # We need that when migration VMs to other nodes (files already moved)
5939 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5940 sub vm_stop {
5941     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5942
5943     $force = 1 if !defined($force) && !$shutdown;
5944
5945     if ($migratedfrom){
5946         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5947         kill 15, $pid if $pid;
5948         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5949         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5950         return;
5951     }
5952
5953     PVE::QemuConfig->lock_config($vmid, sub {
5954         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5955    });
5956 }
5957
5958 sub vm_reboot {
5959     my ($vmid, $timeout) = @_;
5960
5961     PVE::QemuConfig->lock_config($vmid, sub {
5962         eval {
5963
5964             # only reboot if running, as qmeventd starts it again on a stop event
5965             return if !check_running($vmid);
5966
5967             create_reboot_request($vmid);
5968
5969             my $storecfg = PVE::Storage::config();
5970             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5971
5972         };
5973         if (my $err = $@) {
5974             # avoid that the next normal shutdown will be confused for a reboot
5975             clear_reboot_request($vmid);
5976             die $err;
5977         }
5978    });
5979 }
5980
5981 # note: if using the statestorage parameter, the caller has to check privileges
5982 sub vm_suspend {
5983     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5984
5985     my $conf;
5986     my $path;
5987     my $storecfg;
5988     my $vmstate;
5989
5990     PVE::QemuConfig->lock_config($vmid, sub {
5991
5992         $conf = PVE::QemuConfig->load_config($vmid);
5993
5994         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5995         PVE::QemuConfig->check_lock($conf)
5996             if !($skiplock || $is_backing_up);
5997
5998         die "cannot suspend to disk during backup\n"
5999             if $is_backing_up && $includestate;
6000
6001         if ($includestate) {
6002             $conf->{lock} = 'suspending';
6003             my $date = strftime("%Y-%m-%d", localtime(time()));
6004             $storecfg = PVE::Storage::config();
6005             if (!$statestorage) {
6006                 $statestorage = find_vmstate_storage($conf, $storecfg);
6007                 # check permissions for the storage
6008                 my $rpcenv = PVE::RPCEnvironment::get();
6009                 if ($rpcenv->{type} ne 'cli') {
6010                     my $authuser = $rpcenv->get_user();
6011                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6012                 }
6013             }
6014
6015
6016             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6017                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6018             $path = PVE::Storage::path($storecfg, $vmstate);
6019             PVE::QemuConfig->write_config($vmid, $conf);
6020         } else {
6021             mon_cmd($vmid, "stop");
6022         }
6023     });
6024
6025     if ($includestate) {
6026         # save vm state
6027         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6028
6029         eval {
6030             set_migration_caps($vmid, 1);
6031             mon_cmd($vmid, "savevm-start", statefile => $path);
6032             for(;;) {
6033                 my $state = mon_cmd($vmid, "query-savevm");
6034                 if (!$state->{status}) {
6035                     die "savevm not active\n";
6036                 } elsif ($state->{status} eq 'active') {
6037                     sleep(1);
6038                     next;
6039                 } elsif ($state->{status} eq 'completed') {
6040                     print "State saved, quitting\n";
6041                     last;
6042                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6043                     die "query-savevm failed with error '$state->{error}'\n"
6044                 } else {
6045                     die "query-savevm returned status '$state->{status}'\n";
6046                 }
6047             }
6048         };
6049         my $err = $@;
6050
6051         PVE::QemuConfig->lock_config($vmid, sub {
6052             $conf = PVE::QemuConfig->load_config($vmid);
6053             if ($err) {
6054                 # cleanup, but leave suspending lock, to indicate something went wrong
6055                 eval {
6056                     mon_cmd($vmid, "savevm-end");
6057                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6058                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6059                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6060                     PVE::QemuConfig->write_config($vmid, $conf);
6061                 };
6062                 warn $@ if $@;
6063                 die $err;
6064             }
6065
6066             die "lock changed unexpectedly\n"
6067                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6068
6069             mon_cmd($vmid, "quit");
6070             $conf->{lock} = 'suspended';
6071             PVE::QemuConfig->write_config($vmid, $conf);
6072         });
6073     }
6074 }
6075
6076 sub vm_resume {
6077     my ($vmid, $skiplock, $nocheck) = @_;
6078
6079     PVE::QemuConfig->lock_config($vmid, sub {
6080         my $res = mon_cmd($vmid, 'query-status');
6081         my $resume_cmd = 'cont';
6082         my $reset = 0;
6083
6084         if ($res->{status}) {
6085             return if $res->{status} eq 'running'; # job done, go home
6086             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6087             $reset = 1 if $res->{status} eq 'shutdown';
6088         }
6089
6090         if (!$nocheck) {
6091
6092             my $conf = PVE::QemuConfig->load_config($vmid);
6093
6094             PVE::QemuConfig->check_lock($conf)
6095                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6096         }
6097
6098         if ($reset) {
6099             # required if a VM shuts down during a backup and we get a resume
6100             # request before the backup finishes for example
6101             mon_cmd($vmid, "system_reset");
6102         }
6103         mon_cmd($vmid, $resume_cmd);
6104     });
6105 }
6106
6107 sub vm_sendkey {
6108     my ($vmid, $skiplock, $key) = @_;
6109
6110     PVE::QemuConfig->lock_config($vmid, sub {
6111
6112         my $conf = PVE::QemuConfig->load_config($vmid);
6113
6114         # there is no qmp command, so we use the human monitor command
6115         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6116         die $res if $res ne '';
6117     });
6118 }
6119
6120 # vzdump restore implementaion
6121
6122 sub tar_archive_read_firstfile {
6123     my $archive = shift;
6124
6125     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6126
6127     # try to detect archive type first
6128     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6129         die "unable to open file '$archive'\n";
6130     my $firstfile = <$fh>;
6131     kill 15, $pid;
6132     close $fh;
6133
6134     die "ERROR: archive contaions no data\n" if !$firstfile;
6135     chomp $firstfile;
6136
6137     return $firstfile;
6138 }
6139
6140 sub tar_restore_cleanup {
6141     my ($storecfg, $statfile) = @_;
6142
6143     print STDERR "starting cleanup\n";
6144
6145     if (my $fd = IO::File->new($statfile, "r")) {
6146         while (defined(my $line = <$fd>)) {
6147             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6148                 my $volid = $2;
6149                 eval {
6150                     if ($volid =~ m|^/|) {
6151                         unlink $volid || die 'unlink failed\n';
6152                     } else {
6153                         PVE::Storage::vdisk_free($storecfg, $volid);
6154                     }
6155                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6156                 };
6157                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6158             } else {
6159                 print STDERR "unable to parse line in statfile - $line";
6160             }
6161         }
6162         $fd->close();
6163     }
6164 }
6165
6166 sub restore_file_archive {
6167     my ($archive, $vmid, $user, $opts) = @_;
6168
6169     return restore_vma_archive($archive, $vmid, $user, $opts)
6170         if $archive eq '-';
6171
6172     my $info = PVE::Storage::archive_info($archive);
6173     my $format = $opts->{format} // $info->{format};
6174     my $comp = $info->{compression};
6175
6176     # try to detect archive format
6177     if ($format eq 'tar') {
6178         return restore_tar_archive($archive, $vmid, $user, $opts);
6179     } else {
6180         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6181     }
6182 }
6183
6184 # hepler to remove disks that will not be used after restore
6185 my $restore_cleanup_oldconf = sub {
6186     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6187
6188     PVE::QemuConfig->foreach_volume($oldconf, sub {
6189         my ($ds, $drive) = @_;
6190
6191         return if drive_is_cdrom($drive, 1);
6192
6193         my $volid = $drive->{file};
6194         return if !$volid || $volid =~ m|^/|;
6195
6196         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6197         return if !$path || !$owner || ($owner != $vmid);
6198
6199         # Note: only delete disk we want to restore
6200         # other volumes will become unused
6201         if ($virtdev_hash->{$ds}) {
6202             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6203             if (my $err = $@) {
6204                 warn $err;
6205             }
6206         }
6207     });
6208
6209     # delete vmstate files, after the restore we have no snapshots anymore
6210     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6211         my $snap = $oldconf->{snapshots}->{$snapname};
6212         if ($snap->{vmstate}) {
6213             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6214             if (my $err = $@) {
6215                 warn $err;
6216             }
6217         }
6218     }
6219 };
6220
6221 # Helper to parse vzdump backup device hints
6222 #
6223 # $rpcenv: Environment, used to ckeck storage permissions
6224 # $user: User ID, to check storage permissions
6225 # $storecfg: Storage configuration
6226 # $fh: the file handle for reading the configuration
6227 # $devinfo: should contain device sizes for all backu-up'ed devices
6228 # $options: backup options (pool, default storage)
6229 #
6230 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6231 my $parse_backup_hints = sub {
6232     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6233
6234     my $virtdev_hash = {};
6235
6236     while (defined(my $line = <$fh>)) {
6237         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6238             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6239             die "archive does not contain data for drive '$virtdev'\n"
6240                 if !$devinfo->{$devname};
6241
6242             if (defined($options->{storage})) {
6243                 $storeid = $options->{storage} || 'local';
6244             } elsif (!$storeid) {
6245                 $storeid = 'local';
6246             }
6247             $format = 'raw' if !$format;
6248             $devinfo->{$devname}->{devname} = $devname;
6249             $devinfo->{$devname}->{virtdev} = $virtdev;
6250             $devinfo->{$devname}->{format} = $format;
6251             $devinfo->{$devname}->{storeid} = $storeid;
6252
6253             # check permission on storage
6254             my $pool = $options->{pool}; # todo: do we need that?
6255             if ($user ne 'root@pam') {
6256                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6257             }
6258
6259             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6260         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6261             my $virtdev = $1;
6262             my $drive = parse_drive($virtdev, $2);
6263             if (drive_is_cloudinit($drive)) {
6264                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6265                 $storeid = $options->{storage} if defined ($options->{storage});
6266                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6267                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6268
6269                 $virtdev_hash->{$virtdev} = {
6270                     format => $format,
6271                     storeid => $storeid,
6272                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6273                     is_cloudinit => 1,
6274                 };
6275             }
6276         }
6277     }
6278
6279     return $virtdev_hash;
6280 };
6281
6282 # Helper to allocate and activate all volumes required for a restore
6283 #
6284 # $storecfg: Storage configuration
6285 # $virtdev_hash: as returned by parse_backup_hints()
6286 #
6287 # Returns: { $virtdev => $volid }
6288 my $restore_allocate_devices = sub {
6289     my ($storecfg, $virtdev_hash, $vmid) = @_;
6290
6291     my $map = {};
6292     foreach my $virtdev (sort keys %$virtdev_hash) {
6293         my $d = $virtdev_hash->{$virtdev};
6294         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6295         my $storeid = $d->{storeid};
6296         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6297
6298         # test if requested format is supported
6299         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6300         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6301         $d->{format} = $defFormat if !$supported;
6302
6303         my $name;
6304         if ($d->{is_cloudinit}) {
6305             $name = "vm-$vmid-cloudinit";
6306             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6307             if ($scfg->{path}) {
6308                 $name .= ".$d->{format}";
6309             }
6310         }
6311
6312         my $volid = PVE::Storage::vdisk_alloc(
6313             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6314
6315         print STDERR "new volume ID is '$volid'\n";
6316         $d->{volid} = $volid;
6317
6318         PVE::Storage::activate_volumes($storecfg, [$volid]);
6319
6320         $map->{$virtdev} = $volid;
6321     }
6322
6323     return $map;
6324 };
6325
6326 sub restore_update_config_line {
6327     my ($cookie, $map, $line, $unique) = @_;
6328
6329     return '' if $line =~ m/^\#qmdump\#/;
6330     return '' if $line =~ m/^\#vzdump\#/;
6331     return '' if $line =~ m/^lock:/;
6332     return '' if $line =~ m/^unused\d+:/;
6333     return '' if $line =~ m/^parent:/;
6334
6335     my $res = '';
6336
6337     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6338     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6339         # try to convert old 1.X settings
6340         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6341         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6342             my ($model, $macaddr) = split(/\=/, $devconfig);
6343             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6344             my $net = {
6345                 model => $model,
6346                 bridge => "vmbr$ind",
6347                 macaddr => $macaddr,
6348             };
6349             my $netstr = print_net($net);
6350
6351             $res .= "net$cookie->{netcount}: $netstr\n";
6352             $cookie->{netcount}++;
6353         }
6354     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6355         my ($id, $netstr) = ($1, $2);
6356         my $net = parse_net($netstr);
6357         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6358         $netstr = print_net($net);
6359         $res .= "$id: $netstr\n";
6360     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6361         my $virtdev = $1;
6362         my $value = $3;
6363         my $di = parse_drive($virtdev, $value);
6364         if (defined($di->{backup}) && !$di->{backup}) {
6365             $res .= "#$line";
6366         } elsif ($map->{$virtdev}) {
6367             delete $di->{format}; # format can change on restore
6368             $di->{file} = $map->{$virtdev};
6369             $value = print_drive($di);
6370             $res .= "$virtdev: $value\n";
6371         } else {
6372             $res .= $line;
6373         }
6374     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6375         my $vmgenid = $1;
6376         if ($vmgenid ne '0') {
6377             # always generate a new vmgenid if there was a valid one setup
6378             $vmgenid = generate_uuid();
6379         }
6380         $res .= "vmgenid: $vmgenid\n";
6381     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6382         my ($uuid, $uuid_str);
6383         UUID::generate($uuid);
6384         UUID::unparse($uuid, $uuid_str);
6385         my $smbios1 = parse_smbios1($2);
6386         $smbios1->{uuid} = $uuid_str;
6387         $res .= $1.print_smbios1($smbios1)."\n";
6388     } else {
6389         $res .= $line;
6390     }
6391
6392     return $res;
6393 }
6394
6395 my $restore_deactivate_volumes = sub {
6396     my ($storecfg, $devinfo) = @_;
6397
6398     my $vollist = [];
6399     foreach my $devname (keys %$devinfo) {
6400         my $volid = $devinfo->{$devname}->{volid};
6401         push @$vollist, $volid if $volid;
6402     }
6403
6404     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6405 };
6406
6407 my $restore_destroy_volumes = sub {
6408     my ($storecfg, $devinfo) = @_;
6409
6410     foreach my $devname (keys %$devinfo) {
6411         my $volid = $devinfo->{$devname}->{volid};
6412         next if !$volid;
6413         eval {
6414             if ($volid =~ m|^/|) {
6415                 unlink $volid || die 'unlink failed\n';
6416             } else {
6417                 PVE::Storage::vdisk_free($storecfg, $volid);
6418             }
6419             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6420         };
6421         print STDERR "unable to cleanup '$volid' - $@" if $@;
6422     }
6423 };
6424
6425 sub scan_volids {
6426     my ($cfg, $vmid) = @_;
6427
6428     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6429
6430     my $volid_hash = {};
6431     foreach my $storeid (keys %$info) {
6432         foreach my $item (@{$info->{$storeid}}) {
6433             next if !($item->{volid} && $item->{size});
6434             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6435             $volid_hash->{$item->{volid}} = $item;
6436         }
6437     }
6438
6439     return $volid_hash;
6440 }
6441
6442 sub update_disk_config {
6443     my ($vmid, $conf, $volid_hash) = @_;
6444
6445     my $changes;
6446     my $prefix = "VM $vmid";
6447
6448     # used and unused disks
6449     my $referenced = {};
6450
6451     # Note: it is allowed to define multiple storages with same path (alias), so
6452     # we need to check both 'volid' and real 'path' (two different volid can point
6453     # to the same path).
6454
6455     my $referencedpath = {};
6456
6457     # update size info
6458     PVE::QemuConfig->foreach_volume($conf, sub {
6459         my ($opt, $drive) = @_;
6460
6461         my $volid = $drive->{file};
6462         return if !$volid;
6463         my $volume = $volid_hash->{$volid};
6464
6465         # mark volid as "in-use" for next step
6466         $referenced->{$volid} = 1;
6467         if ($volume && (my $path = $volume->{path})) {
6468             $referencedpath->{$path} = 1;
6469         }
6470
6471         return if drive_is_cdrom($drive);
6472         return if !$volume;
6473
6474         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6475         if (defined($updated)) {
6476             $changes = 1;
6477             $conf->{$opt} = print_drive($updated);
6478             print "$prefix ($opt): $msg\n";
6479         }
6480     });
6481
6482     # remove 'unusedX' entry if volume is used
6483     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6484         my ($opt, $drive) = @_;
6485
6486         my $volid = $drive->{file};
6487         return if !$volid;
6488
6489         my $path;
6490         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6491         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6492             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6493             $changes = 1;
6494             delete $conf->{$opt};
6495         }
6496
6497         $referenced->{$volid} = 1;
6498         $referencedpath->{$path} = 1 if $path;
6499     });
6500
6501     foreach my $volid (sort keys %$volid_hash) {
6502         next if $volid =~ m/vm-$vmid-state-/;
6503         next if $referenced->{$volid};
6504         my $path = $volid_hash->{$volid}->{path};
6505         next if !$path; # just to be sure
6506         next if $referencedpath->{$path};
6507         $changes = 1;
6508         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6509         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6510         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6511     }
6512
6513     return $changes;
6514 }
6515
6516 sub rescan {
6517     my ($vmid, $nolock, $dryrun) = @_;
6518
6519     my $cfg = PVE::Storage::config();
6520
6521     print "rescan volumes...\n";
6522     my $volid_hash = scan_volids($cfg, $vmid);
6523
6524     my $updatefn =  sub {
6525         my ($vmid) = @_;
6526
6527         my $conf = PVE::QemuConfig->load_config($vmid);
6528
6529         PVE::QemuConfig->check_lock($conf);
6530
6531         my $vm_volids = {};
6532         foreach my $volid (keys %$volid_hash) {
6533             my $info = $volid_hash->{$volid};
6534             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6535         }
6536
6537         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6538
6539         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6540     };
6541
6542     if (defined($vmid)) {
6543         if ($nolock) {
6544             &$updatefn($vmid);
6545         } else {
6546             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6547         }
6548     } else {
6549         my $vmlist = config_list();
6550         foreach my $vmid (keys %$vmlist) {
6551             if ($nolock) {
6552                 &$updatefn($vmid);
6553             } else {
6554                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6555             }
6556         }
6557     }
6558 }
6559
6560 sub restore_proxmox_backup_archive {
6561     my ($archive, $vmid, $user, $options) = @_;
6562
6563     my $storecfg = PVE::Storage::config();
6564
6565     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6566     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6567
6568     my $fingerprint = $scfg->{fingerprint};
6569     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6570
6571     my $repo = PVE::PBSClient::get_repository($scfg);
6572
6573     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6574     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6575     local $ENV{PBS_PASSWORD} = $password;
6576     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6577
6578     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6579         PVE::Storage::parse_volname($storecfg, $archive);
6580
6581     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6582
6583     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6584
6585     my $tmpdir = "/var/tmp/vzdumptmp$$";
6586     rmtree $tmpdir;
6587     mkpath $tmpdir;
6588
6589     my $conffile = PVE::QemuConfig->config_file($vmid);
6590      # disable interrupts (always do cleanups)
6591     local $SIG{INT} =
6592         local $SIG{TERM} =
6593         local $SIG{QUIT} =
6594         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6595
6596     # Note: $oldconf is undef if VM does not exists
6597     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6598     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6599     my $new_conf_raw = '';
6600
6601     my $rpcenv = PVE::RPCEnvironment::get();
6602     my $devinfo = {};
6603
6604     eval {
6605         # enable interrupts
6606         local $SIG{INT} =
6607             local $SIG{TERM} =
6608             local $SIG{QUIT} =
6609             local $SIG{HUP} =
6610             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6611
6612         my $cfgfn = "$tmpdir/qemu-server.conf";
6613         my $firewall_config_fn = "$tmpdir/fw.conf";
6614         my $index_fn = "$tmpdir/index.json";
6615
6616         my $cmd = "restore";
6617
6618         my $param = [$pbs_backup_name, "index.json", $index_fn];
6619         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6620         my $index = PVE::Tools::file_get_contents($index_fn);
6621         $index = decode_json($index);
6622
6623         # print Dumper($index);
6624         foreach my $info (@{$index->{files}}) {
6625             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6626                 my $devname = $1;
6627                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6628                     $devinfo->{$devname}->{size} = $1;
6629                 } else {
6630                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6631                 }
6632             }
6633         }
6634
6635         my $is_qemu_server_backup = scalar(
6636             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6637         );
6638         if (!$is_qemu_server_backup) {
6639             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6640         }
6641         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6642
6643         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6644         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6645
6646         if ($has_firewall_config) {
6647             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6648             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6649
6650             my $pve_firewall_dir = '/etc/pve/firewall';
6651             mkdir $pve_firewall_dir; # make sure the dir exists
6652             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6653         }
6654
6655         my $fh = IO::File->new($cfgfn, "r") ||
6656             die "unable to read qemu-server.conf - $!\n";
6657
6658         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6659
6660         # fixme: rate limit?
6661
6662         # create empty/temp config
6663         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6664
6665         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6666
6667         # allocate volumes
6668         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6669
6670         foreach my $virtdev (sort keys %$virtdev_hash) {
6671             my $d = $virtdev_hash->{$virtdev};
6672             next if $d->{is_cloudinit}; # no need to restore cloudinit
6673
6674             # this fails if storage is unavailable
6675             my $volid = $d->{volid};
6676             my $path = PVE::Storage::path($storecfg, $volid);
6677
6678             # for live-restore we only want to preload the efidisk and TPM state
6679             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6680
6681             my $pbs_restore_cmd = [
6682                 '/usr/bin/pbs-restore',
6683                 '--repository', $repo,
6684                 $pbs_backup_name,
6685                 "$d->{devname}.img.fidx",
6686                 $path,
6687                 '--verbose',
6688                 ];
6689
6690             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6691             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6692
6693             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6694                 push @$pbs_restore_cmd, '--skip-zero';
6695             }
6696
6697             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6698             print "restore proxmox backup image: $dbg_cmdstring\n";
6699             run_command($pbs_restore_cmd);
6700         }
6701
6702         $fh->seek(0, 0) || die "seek failed - $!\n";
6703
6704         my $cookie = { netcount => 0 };
6705         while (defined(my $line = <$fh>)) {
6706             $new_conf_raw .= restore_update_config_line(
6707                 $cookie,
6708                 $map,
6709                 $line,
6710                 $options->{unique},
6711             );
6712         }
6713
6714         $fh->close();
6715     };
6716     my $err = $@;
6717
6718     if ($err || !$options->{live}) {
6719         $restore_deactivate_volumes->($storecfg, $devinfo);
6720     }
6721
6722     rmtree $tmpdir;
6723
6724     if ($err) {
6725         $restore_destroy_volumes->($storecfg, $devinfo);
6726         die $err;
6727     }
6728
6729     if ($options->{live}) {
6730         # keep lock during live-restore
6731         $new_conf_raw .= "\nlock: create";
6732     }
6733
6734     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6735
6736     PVE::Cluster::cfs_update(); # make sure we read new file
6737
6738     eval { rescan($vmid, 1); };
6739     warn $@ if $@;
6740
6741     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6742
6743     if ($options->{live}) {
6744         # enable interrupts
6745         local $SIG{INT} =
6746             local $SIG{TERM} =
6747             local $SIG{QUIT} =
6748             local $SIG{HUP} =
6749             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6750
6751         my $conf = PVE::QemuConfig->load_config($vmid);
6752         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6753
6754         # these special drives are already restored before start
6755         delete $devinfo->{'drive-efidisk0'};
6756         delete $devinfo->{'drive-tpmstate0-backup'};
6757         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6758
6759         PVE::QemuConfig->remove_lock($vmid, "create");
6760     }
6761 }
6762
6763 sub pbs_live_restore {
6764     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6765
6766     print "starting VM for live-restore\n";
6767     print "repository: '$repo', snapshot: '$snap'\n";
6768
6769     my $pbs_backing = {};
6770     for my $ds (keys %$restored_disks) {
6771         $ds =~ m/^drive-(.*)$/;
6772         my $confname = $1;
6773         $pbs_backing->{$confname} = {
6774             repository => $repo,
6775             snapshot => $snap,
6776             archive => "$ds.img.fidx",
6777         };
6778         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6779
6780         my $drive = parse_drive($confname, $conf->{$confname});
6781         print "restoring '$ds' to '$drive->{file}'\n";
6782     }
6783
6784     my $drives_streamed = 0;
6785     eval {
6786         # make sure HA doesn't interrupt our restore by stopping the VM
6787         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6788             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6789         }
6790
6791         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6792         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6793         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6794
6795         my $qmeventd_fd = register_qmeventd_handle($vmid);
6796
6797         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6798         # this will effectively collapse the backing image chain consisting of
6799         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6800         # removes itself once all backing images vanish with 'auto-remove=on')
6801         my $jobs = {};
6802         for my $ds (sort keys %$restored_disks) {
6803             my $job_id = "restore-$ds";
6804             mon_cmd($vmid, 'block-stream',
6805                 'job-id' => $job_id,
6806                 device => "$ds",
6807             );
6808             $jobs->{$job_id} = {};
6809         }
6810
6811         mon_cmd($vmid, 'cont');
6812         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6813
6814         print "restore-drive jobs finished successfully, removing all tracking block devices"
6815             ." to disconnect from Proxmox Backup Server\n";
6816
6817         for my $ds (sort keys %$restored_disks) {
6818             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6819         }
6820
6821         close($qmeventd_fd);
6822     };
6823
6824     my $err = $@;
6825
6826     if ($err) {
6827         warn "An error occured during live-restore: $err\n";
6828         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6829         die "live-restore failed\n";
6830     }
6831 }
6832
6833 sub restore_vma_archive {
6834     my ($archive, $vmid, $user, $opts, $comp) = @_;
6835
6836     my $readfrom = $archive;
6837
6838     my $cfg = PVE::Storage::config();
6839     my $commands = [];
6840     my $bwlimit = $opts->{bwlimit};
6841
6842     my $dbg_cmdstring = '';
6843     my $add_pipe = sub {
6844         my ($cmd) = @_;
6845         push @$commands, $cmd;
6846         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6847         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6848         $readfrom = '-';
6849     };
6850
6851     my $input = undef;
6852     if ($archive eq '-') {
6853         $input = '<&STDIN';
6854     } else {
6855         # If we use a backup from a PVE defined storage we also consider that
6856         # storage's rate limit:
6857         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6858         if (defined($volid)) {
6859             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6860             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6861             if ($readlimit) {
6862                 print STDERR "applying read rate limit: $readlimit\n";
6863                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6864                 $add_pipe->($cstream);
6865             }
6866         }
6867     }
6868
6869     if ($comp) {
6870         my $info = PVE::Storage::decompressor_info('vma', $comp);
6871         my $cmd = $info->{decompressor};
6872         push @$cmd, $readfrom;
6873         $add_pipe->($cmd);
6874     }
6875
6876     my $tmpdir = "/var/tmp/vzdumptmp$$";
6877     rmtree $tmpdir;
6878
6879     # disable interrupts (always do cleanups)
6880     local $SIG{INT} =
6881         local $SIG{TERM} =
6882         local $SIG{QUIT} =
6883         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6884
6885     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6886     POSIX::mkfifo($mapfifo, 0600);
6887     my $fifofh;
6888     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6889
6890     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6891
6892     my $oldtimeout;
6893     my $timeout = 5;
6894
6895     my $devinfo = {};
6896
6897     my $rpcenv = PVE::RPCEnvironment::get();
6898
6899     my $conffile = PVE::QemuConfig->config_file($vmid);
6900
6901     # Note: $oldconf is undef if VM does not exist
6902     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6903     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6904     my $new_conf_raw = '';
6905
6906     my %storage_limits;
6907
6908     my $print_devmap = sub {
6909         my $cfgfn = "$tmpdir/qemu-server.conf";
6910
6911         # we can read the config - that is already extracted
6912         my $fh = IO::File->new($cfgfn, "r") ||
6913             die "unable to read qemu-server.conf - $!\n";
6914
6915         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6916         if (-f $fwcfgfn) {
6917             my $pve_firewall_dir = '/etc/pve/firewall';
6918             mkdir $pve_firewall_dir; # make sure the dir exists
6919             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6920         }
6921
6922         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6923
6924         foreach my $info (values %{$virtdev_hash}) {
6925             my $storeid = $info->{storeid};
6926             next if defined($storage_limits{$storeid});
6927
6928             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6929             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6930             $storage_limits{$storeid} = $limit * 1024;
6931         }
6932
6933         foreach my $devname (keys %$devinfo) {
6934             die "found no device mapping information for device '$devname'\n"
6935                 if !$devinfo->{$devname}->{virtdev};
6936         }
6937
6938         # create empty/temp config
6939         if ($oldconf) {
6940             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6941             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6942         }
6943
6944         # allocate volumes
6945         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6946
6947         # print restore information to $fifofh
6948         foreach my $virtdev (sort keys %$virtdev_hash) {
6949             my $d = $virtdev_hash->{$virtdev};
6950             next if $d->{is_cloudinit}; # no need to restore cloudinit
6951
6952             my $storeid = $d->{storeid};
6953             my $volid = $d->{volid};
6954
6955             my $map_opts = '';
6956             if (my $limit = $storage_limits{$storeid}) {
6957                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6958             }
6959
6960             my $write_zeros = 1;
6961             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6962                 $write_zeros = 0;
6963             }
6964
6965             my $path = PVE::Storage::path($cfg, $volid);
6966
6967             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6968
6969             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6970         }
6971
6972         $fh->seek(0, 0) || die "seek failed - $!\n";
6973
6974         my $cookie = { netcount => 0 };
6975         while (defined(my $line = <$fh>)) {
6976             $new_conf_raw .= restore_update_config_line(
6977                 $cookie,
6978                 $map,
6979                 $line,
6980                 $opts->{unique},
6981             );
6982         }
6983
6984         $fh->close();
6985     };
6986
6987     eval {
6988         # enable interrupts
6989         local $SIG{INT} =
6990             local $SIG{TERM} =
6991             local $SIG{QUIT} =
6992             local $SIG{HUP} =
6993             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6994         local $SIG{ALRM} = sub { die "got timeout\n"; };
6995
6996         $oldtimeout = alarm($timeout);
6997
6998         my $parser = sub {
6999             my $line = shift;
7000
7001             print "$line\n";
7002
7003             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7004                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7005                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7006             } elsif ($line =~ m/^CTIME: /) {
7007                 # we correctly received the vma config, so we can disable
7008                 # the timeout now for disk allocation (set to 10 minutes, so
7009                 # that we always timeout if something goes wrong)
7010                 alarm(600);
7011                 &$print_devmap();
7012                 print $fifofh "done\n";
7013                 my $tmp = $oldtimeout || 0;
7014                 $oldtimeout = undef;
7015                 alarm($tmp);
7016                 close($fifofh);
7017                 $fifofh = undef;
7018             }
7019         };
7020
7021         print "restore vma archive: $dbg_cmdstring\n";
7022         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7023     };
7024     my $err = $@;
7025
7026     alarm($oldtimeout) if $oldtimeout;
7027
7028     $restore_deactivate_volumes->($cfg, $devinfo);
7029
7030     close($fifofh) if $fifofh;
7031     unlink $mapfifo;
7032     rmtree $tmpdir;
7033
7034     if ($err) {
7035         $restore_destroy_volumes->($cfg, $devinfo);
7036         die $err;
7037     }
7038
7039     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7040
7041     PVE::Cluster::cfs_update(); # make sure we read new file
7042
7043     eval { rescan($vmid, 1); };
7044     warn $@ if $@;
7045
7046     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7047 }
7048
7049 sub restore_tar_archive {
7050     my ($archive, $vmid, $user, $opts) = @_;
7051
7052     if ($archive ne '-') {
7053         my $firstfile = tar_archive_read_firstfile($archive);
7054         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7055             if $firstfile ne 'qemu-server.conf';
7056     }
7057
7058     my $storecfg = PVE::Storage::config();
7059
7060     # avoid zombie disks when restoring over an existing VM -> cleanup first
7061     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7062     # skiplock=1 because qmrestore has set the 'create' lock itself already
7063     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7064     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7065
7066     my $tocmd = "/usr/lib/qemu-server/qmextract";
7067
7068     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7069     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7070     $tocmd .= ' --prealloc' if $opts->{prealloc};
7071     $tocmd .= ' --info' if $opts->{info};
7072
7073     # tar option "xf" does not autodetect compression when read from STDIN,
7074     # so we pipe to zcat
7075     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7076         PVE::Tools::shellquote("--to-command=$tocmd");
7077
7078     my $tmpdir = "/var/tmp/vzdumptmp$$";
7079     mkpath $tmpdir;
7080
7081     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7082     local $ENV{VZDUMP_VMID} = $vmid;
7083     local $ENV{VZDUMP_USER} = $user;
7084
7085     my $conffile = PVE::QemuConfig->config_file($vmid);
7086     my $new_conf_raw = '';
7087
7088     # disable interrupts (always do cleanups)
7089     local $SIG{INT} =
7090         local $SIG{TERM} =
7091         local $SIG{QUIT} =
7092         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7093
7094     eval {
7095         # enable interrupts
7096         local $SIG{INT} =
7097             local $SIG{TERM} =
7098             local $SIG{QUIT} =
7099             local $SIG{HUP} =
7100             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7101
7102         if ($archive eq '-') {
7103             print "extracting archive from STDIN\n";
7104             run_command($cmd, input => "<&STDIN");
7105         } else {
7106             print "extracting archive '$archive'\n";
7107             run_command($cmd);
7108         }
7109
7110         return if $opts->{info};
7111
7112         # read new mapping
7113         my $map = {};
7114         my $statfile = "$tmpdir/qmrestore.stat";
7115         if (my $fd = IO::File->new($statfile, "r")) {
7116             while (defined (my $line = <$fd>)) {
7117                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7118                     $map->{$1} = $2 if $1;
7119                 } else {
7120                     print STDERR "unable to parse line in statfile - $line\n";
7121                 }
7122             }
7123             $fd->close();
7124         }
7125
7126         my $confsrc = "$tmpdir/qemu-server.conf";
7127
7128         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7129
7130         my $cookie = { netcount => 0 };
7131         while (defined (my $line = <$srcfd>)) {
7132             $new_conf_raw .= restore_update_config_line(
7133                 $cookie,
7134                 $map,
7135                 $line,
7136                 $opts->{unique},
7137             );
7138         }
7139
7140         $srcfd->close();
7141     };
7142     if (my $err = $@) {
7143         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7144         die $err;
7145     }
7146
7147     rmtree $tmpdir;
7148
7149     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7150
7151     PVE::Cluster::cfs_update(); # make sure we read new file
7152
7153     eval { rescan($vmid, 1); };
7154     warn $@ if $@;
7155 };
7156
7157 sub foreach_storage_used_by_vm {
7158     my ($conf, $func) = @_;
7159
7160     my $sidhash = {};
7161
7162     PVE::QemuConfig->foreach_volume($conf, sub {
7163         my ($ds, $drive) = @_;
7164         return if drive_is_cdrom($drive);
7165
7166         my $volid = $drive->{file};
7167
7168         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7169         $sidhash->{$sid} = $sid if $sid;
7170     });
7171
7172     foreach my $sid (sort keys %$sidhash) {
7173         &$func($sid);
7174     }
7175 }
7176
7177 my $qemu_snap_storage = {
7178     rbd => 1,
7179 };
7180 sub do_snapshots_with_qemu {
7181     my ($storecfg, $volid, $deviceid) = @_;
7182
7183     return if $deviceid =~ m/tpmstate0/;
7184
7185     my $storage_name = PVE::Storage::parse_volume_id($volid);
7186     my $scfg = $storecfg->{ids}->{$storage_name};
7187     die "could not find storage '$storage_name'\n" if !defined($scfg);
7188
7189     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7190         return 1;
7191     }
7192
7193     if ($volid =~ m/\.(qcow2|qed)$/){
7194         return 1;
7195     }
7196
7197     return;
7198 }
7199
7200 sub qga_check_running {
7201     my ($vmid, $nowarn) = @_;
7202
7203     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7204     if ($@) {
7205         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7206         return 0;
7207     }
7208     return 1;
7209 }
7210
7211 sub template_create {
7212     my ($vmid, $conf, $disk) = @_;
7213
7214     my $storecfg = PVE::Storage::config();
7215
7216     PVE::QemuConfig->foreach_volume($conf, sub {
7217         my ($ds, $drive) = @_;
7218
7219         return if drive_is_cdrom($drive);
7220         return if $disk && $ds ne $disk;
7221
7222         my $volid = $drive->{file};
7223         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7224
7225         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7226         $drive->{file} = $voliddst;
7227         $conf->{$ds} = print_drive($drive);
7228         PVE::QemuConfig->write_config($vmid, $conf);
7229     });
7230 }
7231
7232 sub convert_iscsi_path {
7233     my ($path) = @_;
7234
7235     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7236         my $portal = $1;
7237         my $target = $2;
7238         my $lun = $3;
7239
7240         my $initiator_name = get_initiator_name();
7241
7242         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7243                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7244     }
7245
7246     die "cannot convert iscsi path '$path', unkown format\n";
7247 }
7248
7249 sub qemu_img_convert {
7250     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7251
7252     my $storecfg = PVE::Storage::config();
7253     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7254     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7255
7256     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7257
7258     my $cachemode;
7259     my $src_path;
7260     my $src_is_iscsi = 0;
7261     my $src_format;
7262
7263     if ($src_storeid) {
7264         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7265         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7266         $src_format = qemu_img_format($src_scfg, $src_volname);
7267         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7268         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7269         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7270     } elsif (-f $src_volid) {
7271         $src_path = $src_volid;
7272         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7273             $src_format = $1;
7274         }
7275     }
7276
7277     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7278
7279     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7280     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7281     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7282     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7283
7284     my $cmd = [];
7285     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7286     push @$cmd, '-l', "snapshot.name=$snapname"
7287         if $snapname && $src_format && $src_format eq "qcow2";
7288     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7289     push @$cmd, '-T', $cachemode if defined($cachemode);
7290
7291     if ($src_is_iscsi) {
7292         push @$cmd, '--image-opts';
7293         $src_path = convert_iscsi_path($src_path);
7294     } elsif ($src_format) {
7295         push @$cmd, '-f', $src_format;
7296     }
7297
7298     if ($dst_is_iscsi) {
7299         push @$cmd, '--target-image-opts';
7300         $dst_path = convert_iscsi_path($dst_path);
7301     } else {
7302         push @$cmd, '-O', $dst_format;
7303     }
7304
7305     push @$cmd, $src_path;
7306
7307     if (!$dst_is_iscsi && $is_zero_initialized) {
7308         push @$cmd, "zeroinit:$dst_path";
7309     } else {
7310         push @$cmd, $dst_path;
7311     }
7312
7313     my $parser = sub {
7314         my $line = shift;
7315         if($line =~ m/\((\S+)\/100\%\)/){
7316             my $percent = $1;
7317             my $transferred = int($size * $percent / 100);
7318             my $total_h = render_bytes($size, 1);
7319             my $transferred_h = render_bytes($transferred, 1);
7320
7321             print "transferred $transferred_h of $total_h ($percent%)\n";
7322         }
7323
7324     };
7325
7326     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7327     my $err = $@;
7328     die "copy failed: $err" if $err;
7329 }
7330
7331 sub qemu_img_format {
7332     my ($scfg, $volname) = @_;
7333
7334     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7335         return $1;
7336     } else {
7337         return "raw";
7338     }
7339 }
7340
7341 sub qemu_drive_mirror {
7342     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7343
7344     $jobs = {} if !$jobs;
7345
7346     my $qemu_target;
7347     my $format;
7348     $jobs->{"drive-$drive"} = {};
7349
7350     if ($dst_volid =~ /^nbd:/) {
7351         $qemu_target = $dst_volid;
7352         $format = "nbd";
7353     } else {
7354         my $storecfg = PVE::Storage::config();
7355         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7356
7357         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7358
7359         $format = qemu_img_format($dst_scfg, $dst_volname);
7360
7361         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7362
7363         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7364     }
7365
7366     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7367     $opts->{format} = $format if $format;
7368
7369     if (defined($src_bitmap)) {
7370         $opts->{sync} = 'incremental';
7371         $opts->{bitmap} = $src_bitmap;
7372         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7373     }
7374
7375     if (defined($bwlimit)) {
7376         $opts->{speed} = $bwlimit * 1024;
7377         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7378     } else {
7379         print "drive mirror is starting for drive-$drive\n";
7380     }
7381
7382     # if a job already runs for this device we get an error, catch it for cleanup
7383     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7384     if (my $err = $@) {
7385         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7386         warn "$@\n" if $@;
7387         die "mirroring error: $err\n";
7388     }
7389
7390     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7391 }
7392
7393 # $completion can be either
7394 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7395 # 'cancel': wait until all jobs are ready, block-job-cancel them
7396 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7397 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7398 sub qemu_drive_mirror_monitor {
7399     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7400
7401     $completion //= 'complete';
7402     $op //= "mirror";
7403
7404     eval {
7405         my $err_complete = 0;
7406
7407         my $starttime = time ();
7408         while (1) {
7409             die "block job ('$op') timed out\n" if $err_complete > 300;
7410
7411             my $stats = mon_cmd($vmid, "query-block-jobs");
7412             my $ctime = time();
7413
7414             my $running_jobs = {};
7415             for my $stat (@$stats) {
7416                 next if $stat->{type} ne $op;
7417                 $running_jobs->{$stat->{device}} = $stat;
7418             }
7419
7420             my $readycounter = 0;
7421
7422             for my $job_id (sort keys %$jobs) {
7423                 my $job = $running_jobs->{$job_id};
7424
7425                 my $vanished = !defined($job);
7426                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7427                 if($complete || ($vanished && $completion eq 'auto')) {
7428                     print "$job_id: $op-job finished\n";
7429                     delete $jobs->{$job_id};
7430                     next;
7431                 }
7432
7433                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7434
7435                 my $busy = $job->{busy};
7436                 my $ready = $job->{ready};
7437                 if (my $total = $job->{len}) {
7438                     my $transferred = $job->{offset} || 0;
7439                     my $remaining = $total - $transferred;
7440                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7441
7442                     my $duration = $ctime - $starttime;
7443                     my $total_h = render_bytes($total, 1);
7444                     my $transferred_h = render_bytes($transferred, 1);
7445
7446                     my $status = sprintf(
7447                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7448                         render_duration($duration),
7449                     );
7450
7451                     if ($ready) {
7452                         if ($busy) {
7453                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7454                         } else {
7455                             $status .= ", ready";
7456                         }
7457                     }
7458                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7459                     $jobs->{$job_id}->{ready} = $ready;
7460                 }
7461
7462                 $readycounter++ if $job->{ready};
7463             }
7464
7465             last if scalar(keys %$jobs) == 0;
7466
7467             if ($readycounter == scalar(keys %$jobs)) {
7468                 print "all '$op' jobs are ready\n";
7469
7470                 # do the complete later (or has already been done)
7471                 last if $completion eq 'skip' || $completion eq 'auto';
7472
7473                 if ($vmiddst && $vmiddst != $vmid) {
7474                     my $agent_running = $qga && qga_check_running($vmid);
7475                     if ($agent_running) {
7476                         print "freeze filesystem\n";
7477                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7478                         warn $@ if $@;
7479                     } else {
7480                         print "suspend vm\n";
7481                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7482                         warn $@ if $@;
7483                     }
7484
7485                     # if we clone a disk for a new target vm, we don't switch the disk
7486                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7487
7488                     if ($agent_running) {
7489                         print "unfreeze filesystem\n";
7490                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7491                         warn $@ if $@;
7492                     } else {
7493                         print "resume vm\n";
7494                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7495                         warn $@ if $@;
7496                     }
7497
7498                     last;
7499                 } else {
7500
7501                     for my $job_id (sort keys %$jobs) {
7502                         # try to switch the disk if source and destination are on the same guest
7503                         print "$job_id: Completing block job_id...\n";
7504
7505                         my $op;
7506                         if ($completion eq 'complete') {
7507                             $op = 'block-job-complete';
7508                         } elsif ($completion eq 'cancel') {
7509                             $op = 'block-job-cancel';
7510                         } else {
7511                             die "invalid completion value: $completion\n";
7512                         }
7513                         eval { mon_cmd($vmid, $op, device => $job_id) };
7514                         if ($@ =~ m/cannot be completed/) {
7515                             print "$job_id: block job cannot be completed, trying again.\n";
7516                             $err_complete++;
7517                         }else {
7518                             print "$job_id: Completed successfully.\n";
7519                             $jobs->{$job_id}->{complete} = 1;
7520                         }
7521                     }
7522                 }
7523             }
7524             sleep 1;
7525         }
7526     };
7527     my $err = $@;
7528
7529     if ($err) {
7530         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7531         die "block job ($op) error: $err";
7532     }
7533 }
7534
7535 sub qemu_blockjobs_cancel {
7536     my ($vmid, $jobs) = @_;
7537
7538     foreach my $job (keys %$jobs) {
7539         print "$job: Cancelling block job\n";
7540         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7541         $jobs->{$job}->{cancel} = 1;
7542     }
7543
7544     while (1) {
7545         my $stats = mon_cmd($vmid, "query-block-jobs");
7546
7547         my $running_jobs = {};
7548         foreach my $stat (@$stats) {
7549             $running_jobs->{$stat->{device}} = $stat;
7550         }
7551
7552         foreach my $job (keys %$jobs) {
7553
7554             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7555                 print "$job: Done.\n";
7556                 delete $jobs->{$job};
7557             }
7558         }
7559
7560         last if scalar(keys %$jobs) == 0;
7561
7562         sleep 1;
7563     }
7564 }
7565
7566 sub clone_disk {
7567     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7568         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7569
7570     my $newvolid;
7571
7572     if (!$full) {
7573         print "create linked clone of drive $drivename ($drive->{file})\n";
7574         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7575         push @$newvollist, $newvolid;
7576     } else {
7577
7578         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7579         $storeid = $storage if $storage;
7580
7581         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7582
7583         print "create full clone of drive $drivename ($drive->{file})\n";
7584         my $name = undef;
7585         my $size = undef;
7586         if (drive_is_cloudinit($drive)) {
7587             $name = "vm-$newvmid-cloudinit";
7588             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7589             if ($scfg->{path}) {
7590                 $name .= ".$dst_format";
7591             }
7592             $snapname = undef;
7593             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7594         } elsif ($drivename eq 'efidisk0') {
7595             $size = get_efivars_size($conf);
7596         } elsif ($drivename eq 'tpmstate0') {
7597             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7598         } else {
7599             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7600         }
7601         $newvolid = PVE::Storage::vdisk_alloc(
7602             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7603         );
7604         push @$newvollist, $newvolid;
7605
7606         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7607
7608         if (drive_is_cloudinit($drive)) {
7609             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7610             # if this is the case, we have to complete any block-jobs still there from
7611             # previous drive-mirrors
7612             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7613                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7614             }
7615             goto no_data_clone;
7616         }
7617
7618         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7619         if (!$running || $snapname) {
7620             # TODO: handle bwlimits
7621             if ($drivename eq 'efidisk0') {
7622                 # the relevant data on the efidisk may be smaller than the source
7623                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7624                 # that is given by the OVMF_VARS.fd
7625                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7626                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7627
7628                 # better for Ceph if block size is not too small, see bug #3324
7629                 my $bs = 1024*1024;
7630
7631                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7632                     "if=$src_path", "of=$dst_path"]);
7633             } else {
7634                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7635             }
7636         } else {
7637
7638             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7639
7640             my $kvmver = get_running_qemu_version ($vmid);
7641             if (!min_version($kvmver, 2, 7)) {
7642                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7643                     if $drive->{iothread};
7644             }
7645
7646             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7647                 $completion, $qga, $bwlimit);
7648         }
7649     }
7650
7651 no_data_clone:
7652     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7653
7654     my $disk = dclone($drive);
7655     delete $disk->{format};
7656     $disk->{file} = $newvolid;
7657     $disk->{size} = $size if defined($size);
7658
7659     return $disk;
7660 }
7661
7662 sub get_running_qemu_version {
7663     my ($vmid) = @_;
7664     my $res = mon_cmd($vmid, "query-version");
7665     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7666 }
7667
7668 sub qemu_use_old_bios_files {
7669     my ($machine_type) = @_;
7670
7671     return if !$machine_type;
7672
7673     my $use_old_bios_files = undef;
7674
7675     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7676         $machine_type = $1;
7677         $use_old_bios_files = 1;
7678     } else {
7679         my $version = extract_version($machine_type, kvm_user_version());
7680         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7681         # load new efi bios files on migration. So this hack is required to allow
7682         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7683         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7684         $use_old_bios_files = !min_version($version, 2, 4);
7685     }
7686
7687     return ($use_old_bios_files, $machine_type);
7688 }
7689
7690 sub get_efivars_size {
7691     my ($conf) = @_;
7692     my $arch = get_vm_arch($conf);
7693     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7694     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7695     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7696     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7697     return -s $ovmf_vars;
7698 }
7699
7700 sub update_efidisk_size {
7701     my ($conf) = @_;
7702
7703     return if !defined($conf->{efidisk0});
7704
7705     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7706     $disk->{size} = get_efivars_size($conf);
7707     $conf->{efidisk0} = print_drive($disk);
7708
7709     return;
7710 }
7711
7712 sub update_tpmstate_size {
7713     my ($conf) = @_;
7714
7715     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7716     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7717     $conf->{tpmstate0} = print_drive($disk);
7718 }
7719
7720 sub create_efidisk($$$$$$$) {
7721     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7722
7723     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7724     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7725
7726     my $vars_size_b = -s $ovmf_vars;
7727     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7728     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7729     PVE::Storage::activate_volumes($storecfg, [$volid]);
7730
7731     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7732     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7733
7734     return ($volid, $size/1024);
7735 }
7736
7737 sub vm_iothreads_list {
7738     my ($vmid) = @_;
7739
7740     my $res = mon_cmd($vmid, 'query-iothreads');
7741
7742     my $iothreads = {};
7743     foreach my $iothread (@$res) {
7744         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7745     }
7746
7747     return $iothreads;
7748 }
7749
7750 sub scsihw_infos {
7751     my ($conf, $drive) = @_;
7752
7753     my $maxdev = 0;
7754
7755     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7756         $maxdev = 7;
7757     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7758         $maxdev = 1;
7759     } else {
7760         $maxdev = 256;
7761     }
7762
7763     my $controller = int($drive->{index} / $maxdev);
7764     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7765         ? "virtioscsi"
7766         : "scsihw";
7767
7768     return ($maxdev, $controller, $controller_prefix);
7769 }
7770
7771 sub windows_version {
7772     my ($ostype) = @_;
7773
7774     return 0 if !$ostype;
7775
7776     my $winversion = 0;
7777
7778     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7779         $winversion = 5;
7780     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7781         $winversion = 6;
7782     } elsif ($ostype =~ m/^win(\d+)$/) {
7783         $winversion = $1;
7784     }
7785
7786     return $winversion;
7787 }
7788
7789 sub resolve_dst_disk_format {
7790         my ($storecfg, $storeid, $src_volname, $format) = @_;
7791         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7792
7793         if (!$format) {
7794             # if no target format is specified, use the source disk format as hint
7795             if ($src_volname) {
7796                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7797                 $format = qemu_img_format($scfg, $src_volname);
7798             } else {
7799                 return $defFormat;
7800             }
7801         }
7802
7803         # test if requested format is supported - else use default
7804         my $supported = grep { $_ eq $format } @$validFormats;
7805         $format = $defFormat if !$supported;
7806         return $format;
7807 }
7808
7809 # NOTE: if this logic changes, please update docs & possibly gui logic
7810 sub find_vmstate_storage {
7811     my ($conf, $storecfg) = @_;
7812
7813     # first, return storage from conf if set
7814     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7815
7816     my ($target, $shared, $local);
7817
7818     foreach_storage_used_by_vm($conf, sub {
7819         my ($sid) = @_;
7820         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7821         my $dst = $scfg->{shared} ? \$shared : \$local;
7822         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7823     });
7824
7825     # second, use shared storage where VM has at least one disk
7826     # third, use local storage where VM has at least one disk
7827     # fall back to local storage
7828     $target = $shared // $local // 'local';
7829
7830     return $target;
7831 }
7832
7833 sub generate_uuid {
7834     my ($uuid, $uuid_str);
7835     UUID::generate($uuid);
7836     UUID::unparse($uuid, $uuid_str);
7837     return $uuid_str;
7838 }
7839
7840 sub generate_smbios1_uuid {
7841     return "uuid=".generate_uuid();
7842 }
7843
7844 sub nbd_stop {
7845     my ($vmid) = @_;
7846
7847     mon_cmd($vmid, 'nbd-server-stop');
7848 }
7849
7850 sub create_reboot_request {
7851     my ($vmid) = @_;
7852     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7853         or die "failed to create reboot trigger file: $!\n";
7854     close($fh);
7855 }
7856
7857 sub clear_reboot_request {
7858     my ($vmid) = @_;
7859     my $path = "/run/qemu-server/$vmid.reboot";
7860     my $res = 0;
7861
7862     $res = unlink($path);
7863     die "could not remove reboot request for $vmid: $!"
7864         if !$res && $! != POSIX::ENOENT;
7865
7866     return $res;
7867 }
7868
7869 sub bootorder_from_legacy {
7870     my ($conf, $bootcfg) = @_;
7871
7872     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7873     my $bootindex_hash = {};
7874     my $i = 1;
7875     foreach my $o (split(//, $boot)) {
7876         $bootindex_hash->{$o} = $i*100;
7877         $i++;
7878     }
7879
7880     my $bootorder = {};
7881
7882     PVE::QemuConfig->foreach_volume($conf, sub {
7883         my ($ds, $drive) = @_;
7884
7885         if (drive_is_cdrom ($drive, 1)) {
7886             if ($bootindex_hash->{d}) {
7887                 $bootorder->{$ds} = $bootindex_hash->{d};
7888                 $bootindex_hash->{d} += 1;
7889             }
7890         } elsif ($bootindex_hash->{c}) {
7891             $bootorder->{$ds} = $bootindex_hash->{c}
7892                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7893             $bootindex_hash->{c} += 1;
7894         }
7895     });
7896
7897     if ($bootindex_hash->{n}) {
7898         for (my $i = 0; $i < $MAX_NETS; $i++) {
7899             my $netname = "net$i";
7900             next if !$conf->{$netname};
7901             $bootorder->{$netname} = $bootindex_hash->{n};
7902             $bootindex_hash->{n} += 1;
7903         }
7904     }
7905
7906     return $bootorder;
7907 }
7908
7909 # Generate default device list for 'boot: order=' property. Matches legacy
7910 # default boot order, but with explicit device names. This is important, since
7911 # the fallback for when neither 'order' nor the old format is specified relies
7912 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7913 sub get_default_bootdevices {
7914     my ($conf) = @_;
7915
7916     my @ret = ();
7917
7918     # harddisk
7919     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7920     push @ret, $first if $first;
7921
7922     # cdrom
7923     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7924     push @ret, $first if $first;
7925
7926     # network
7927     for (my $i = 0; $i < $MAX_NETS; $i++) {
7928         my $netname = "net$i";
7929         next if !$conf->{$netname};
7930         push @ret, $netname;
7931         last;
7932     }
7933
7934     return \@ret;
7935 }
7936
7937 sub device_bootorder {
7938     my ($conf) = @_;
7939
7940     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7941
7942     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7943
7944     my $bootorder = {};
7945     if (!defined($boot) || $boot->{legacy}) {
7946         $bootorder = bootorder_from_legacy($conf, $boot);
7947     } elsif ($boot->{order}) {
7948         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7949         for my $dev (PVE::Tools::split_list($boot->{order})) {
7950             $bootorder->{$dev} = $i++;
7951         }
7952     }
7953
7954     return $bootorder;
7955 }
7956
7957 sub register_qmeventd_handle {
7958     my ($vmid) = @_;
7959
7960     my $fh;
7961     my $peer = "/var/run/qmeventd.sock";
7962     my $count = 0;
7963
7964     for (;;) {
7965         $count++;
7966         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7967         last if $fh;
7968         if ($! != EINTR && $! != EAGAIN) {
7969             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7970         }
7971         if ($count > 4) {
7972             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7973               . "after $count retries\n";
7974         }
7975         usleep(25000);
7976     }
7977
7978     # send handshake to mark VM as backing up
7979     print $fh to_json({vzdump => {vmid => "$vmid"}});
7980
7981     # return handle to be closed later when inhibit is no longer required
7982     return $fh;
7983 }
7984
7985 # bash completion helper
7986
7987 sub complete_backup_archives {
7988     my ($cmdname, $pname, $cvalue) = @_;
7989
7990     my $cfg = PVE::Storage::config();
7991
7992     my $storeid;
7993
7994     if ($cvalue =~ m/^([^:]+):/) {
7995         $storeid = $1;
7996     }
7997
7998     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7999
8000     my $res = [];
8001     foreach my $id (keys %$data) {
8002         foreach my $item (@{$data->{$id}}) {
8003             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8004             push @$res, $item->{volid} if defined($item->{volid});
8005         }
8006     }
8007
8008     return $res;
8009 }
8010
8011 my $complete_vmid_full = sub {
8012     my ($running) = @_;
8013
8014     my $idlist = vmstatus();
8015
8016     my $res = [];
8017
8018     foreach my $id (keys %$idlist) {
8019         my $d = $idlist->{$id};
8020         if (defined($running)) {
8021             next if $d->{template};
8022             next if $running && $d->{status} ne 'running';
8023             next if !$running && $d->{status} eq 'running';
8024         }
8025         push @$res, $id;
8026
8027     }
8028     return $res;
8029 };
8030
8031 sub complete_vmid {
8032     return &$complete_vmid_full();
8033 }
8034
8035 sub complete_vmid_stopped {
8036     return &$complete_vmid_full(0);
8037 }
8038
8039 sub complete_vmid_running {
8040     return &$complete_vmid_full(1);
8041 }
8042
8043 sub complete_storage {
8044
8045     my $cfg = PVE::Storage::config();
8046     my $ids = $cfg->{ids};
8047
8048     my $res = [];
8049     foreach my $sid (keys %$ids) {
8050         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8051         next if !$ids->{$sid}->{content}->{images};
8052         push @$res, $sid;
8053     }
8054
8055     return $res;
8056 }
8057
8058 sub complete_migration_storage {
8059     my ($cmd, $param, $current_value, $all_args) = @_;
8060
8061     my $targetnode = @$all_args[1];
8062
8063     my $cfg = PVE::Storage::config();
8064     my $ids = $cfg->{ids};
8065
8066     my $res = [];
8067     foreach my $sid (keys %$ids) {
8068         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8069         next if !$ids->{$sid}->{content}->{images};
8070         push @$res, $sid;
8071     }
8072
8073     return $res;
8074 }
8075
8076 sub vm_is_paused {
8077     my ($vmid) = @_;
8078     my $qmpstatus = eval {
8079         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8080         mon_cmd($vmid, "query-status");
8081     };
8082     warn "$@\n" if $@;
8083     return $qmpstatus && $qmpstatus->{status} eq "paused";
8084 }
8085
8086 sub check_volume_storage_type {
8087     my ($storecfg, $vol) = @_;
8088
8089     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8090     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8091     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8092
8093     die "storage '$storeid' does not support content-type '$vtype'\n"
8094         if !$scfg->{content}->{$vtype};
8095
8096     return 1;
8097 }
8098
8099 1;