PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 123     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 124     type => 'string',
 125     format => 'storage-pair-list',
 126     optional => 1,
 127 });
 128
 129 #no warnings 'redefine';
 130
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0
 167     },
 168     type => {
 169         description => "Select the agent type",
 170         type => 'string',
 171         default => 'virtio',
 172         optional => 1,
 173         enum => [qw(virtio isa)],
 174     },
 175 };
 176
 177 my $vga_fmt = {
 178     type => {
 179         description => "Select the VGA type.",
 180         type => 'string',
 181         default => 'std',
 182         optional => 1,
 183         default_key => 1,
 184         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 185     },
 186     memory => {
 187         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 188         type => 'integer',
 189         optional => 1,
 190         minimum => 4,
 191         maximum => 512,
 192     },
 193 };
 194
 195 my $ivshmem_fmt = {
 196     size => {
 197         type => 'integer',
 198         minimum => 1,
 199         description => "The size of the file in MB.",
 200     },
 201     name => {
 202         type => 'string',
 203         pattern => '[a-zA-Z0-9\-]+',
 204         optional => 1,
 205         format_description => 'string',
 206         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 207     },
 208 };
 209
 210 my $audio_fmt = {
 211     device => {
 212         type => 'string',
 213         enum => [qw(ich9-intel-hda intel-hda AC97)],
 214         description =>  "Configure an audio device."
 215     },
 216     driver =>  {
 217         type => 'string',
 218         enum => ['spice', 'none'],
 219         default => 'spice',
 220         optional => 1,
 221         description => "Driver backend for the audio device."
 222     },
 223 };
 224
 225 my $spice_enhancements_fmt = {
 226     foldersharing => {
 227         type => 'boolean',
 228         optional => 1,
 229         default => '0',
 230         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 231     },
 232     videostreaming =>  {
 233         type => 'string',
 234         enum => ['off', 'all', 'filter'],
 235         default => 'off',
 236         optional => 1,
 237         description => "Enable video streaming. Uses compression for detected video streams."
 238     },
 239 };
 240
 241 my $rng_fmt = {
 242     source => {
 243         type => 'string',
 244         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 245         default_key => 1,
 246         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 247             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 248             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 249             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 250             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 251             ." a hardware RNG from the host.",
 252     },
 253     max_bytes => {
 254         type => 'integer',
 255         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 256             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 257             ." `0` to disable limiting (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 261         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 262         # reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 268             ." the guest to retrieve another 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $meta_info_fmt = {
 275     'ctime' => {
 276         type => 'integer',
 277         description => "The guest creation timestamp as UNIX epoch time",
 278         minimum => 0,
 279         optional => 1,
 280     },
 281     'creation-qemu' => {
 282         type => 'string',
 283         description => "The QEMU (machine) version from the time this VM was created.",
 284         pattern => '\d+(\.\d+)+',
 285         optional => 1,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 2,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. The default is read from the"
 371             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710     meta => {
 711         type => 'string',
 712         format => $meta_info_fmt,
 713         description => "Some (read-only) meta-information about this guest.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.",
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.",
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037     my ($volid, $noerr) = @_;
1038
1039     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1040         return $volid;
1041     }
1042
1043     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1044     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1045     if ($@) {
1046         return if $noerr;
1047         die $@;
1048     }
1049     return $volid;
1050 }
1051
1052 my $usb_fmt = {
1053     host => {
1054         default_key => 1,
1055         type => 'string', format => 'pve-qm-usb-device',
1056         format_description => 'HOSTUSBDEVICE|spice',
1057         description => <<EODESCR,
1058 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1059
1060  'bus-port(.port)*' (decimal numbers) or
1061  'vendor_id:product_id' (hexadeciaml numbers) or
1062  'spice'
1063
1064 You can use the 'lsusb -t' command to list existing usb devices.
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 The value 'spice' can be used to add a usb redirection devices for spice.
1070 EODESCR
1071     },
1072     usb3 => {
1073         optional => 1,
1074         type => 'boolean',
1075         description => "Specifies whether if given host option is a USB3 device or port.",
1076         default => 0,
1077     },
1078 };
1079
1080 my $usbdesc = {
1081     optional => 1,
1082     type => 'string', format => $usb_fmt,
1083     description => "Configure an USB device (n is 0 to 4).",
1084 };
1085 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1086
1087 my $serialdesc = {
1088         optional => 1,
1089         type => 'string',
1090         pattern => '(/dev/.+|socket)',
1091         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1092         verbose_description =>  <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1096
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1098 use with special care.
1099
1100 CAUTION: Experimental! User reported problems with this option.
1101 EODESCR
1102 };
1103
1104 my $paralleldesc= {
1105         optional => 1,
1106         type => 'string',
1107         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1108         description =>  "Map host parallel devices (n is 0 to 2).",
1109         verbose_description =>  <<EODESCR,
1110 Map host parallel devices (n is 0 to 2).
1111
1112 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1113 machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1129 }
1130
1131 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1132     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1133 }
1134
1135 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1136     $confdesc->{"usb$i"} = $usbdesc;
1137 }
1138
1139 my $boot_fmt = {
1140     legacy => {
1141         optional => 1,
1142         default_key => 1,
1143         type => 'string',
1144         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1145                      . " Deprecated, use 'order=' instead.",
1146         pattern => '[acdn]{1,4}',
1147         format_description => "[acdn]{1,4}",
1148
1149         # note: this is also the fallback if boot: is not given at all
1150         default => 'cdn',
1151     },
1152     order => {
1153         optional => 1,
1154         type => 'string',
1155         format => 'pve-qm-bootdev-list',
1156         format_description => "device[;device...]",
1157         description => <<EODESC,
1158 The guest will attempt to boot from devices in the order they appear here.
1159
1160 Disks, optical drives and passed-through storage USB devices will be directly
1161 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1162 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1163
1164 Note that only devices in this list will be marked as bootable and thus loaded
1165 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1166 (e.g. software-raid), you need to specify all of them here.
1167
1168 Overrides the deprecated 'legacy=[acdn]*' value when given.
1169 EODESC
1170     },
1171 };
1172 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1173
1174 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1175 sub verify_bootdev {
1176     my ($dev, $noerr) = @_;
1177
1178     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1179     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1180
1181     my $check = sub {
1182         my ($base) = @_;
1183         return 0 if $dev !~ m/^$base\d+$/;
1184         return 0 if !$confdesc->{$dev};
1185         return 1;
1186     };
1187
1188     return $dev if $check->("net");
1189     return $dev if $check->("usb");
1190     return $dev if $check->("hostpci");
1191
1192     return if $noerr;
1193     die "invalid boot device '$dev'\n";
1194 }
1195
1196 sub print_bootorder {
1197     my ($devs) = @_;
1198     return "" if !@$devs;
1199     my $data = { order => join(';', @$devs) };
1200     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1201 }
1202
1203 my $kvm_api_version = 0;
1204
1205 sub kvm_version {
1206     return $kvm_api_version if $kvm_api_version;
1207
1208     open my $fh, '<', '/dev/kvm' or return;
1209
1210     # 0xae00 => KVM_GET_API_VERSION
1211     $kvm_api_version = ioctl($fh, 0xae00, 0);
1212     close($fh);
1213
1214     return $kvm_api_version;
1215 }
1216
1217 my $kvm_user_version = {};
1218 my $kvm_mtime = {};
1219
1220 sub kvm_user_version {
1221     my ($binary) = @_;
1222
1223     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1224     my $st = stat($binary);
1225
1226     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1227     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1228         $cachedmtime == $st->mtime;
1229
1230     $kvm_user_version->{$binary} = 'unknown';
1231     $kvm_mtime->{$binary} = $st->mtime;
1232
1233     my $code = sub {
1234         my $line = shift;
1235         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1236             $kvm_user_version->{$binary} = $2;
1237         }
1238     };
1239
1240     eval { run_command([$binary, '--version'], outfunc => $code); };
1241     warn $@ if $@;
1242
1243     return $kvm_user_version->{$binary};
1244
1245 }
1246 my sub extract_version {
1247     my ($machine_type, $version) = @_;
1248     $version = kvm_user_version() if !defined($version);
1249     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1250 }
1251
1252 sub kernel_has_vhost_net {
1253     return -c '/dev/vhost-net';
1254 }
1255
1256 sub option_exists {
1257     my $key = shift;
1258     return defined($confdesc->{$key});
1259 }
1260
1261 my $cdrom_path;
1262 sub get_cdrom_path {
1263
1264     return  $cdrom_path if $cdrom_path;
1265
1266     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1267     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1268     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1269 }
1270
1271 sub get_iso_path {
1272     my ($storecfg, $vmid, $cdrom) = @_;
1273
1274     if ($cdrom eq 'cdrom') {
1275         return get_cdrom_path();
1276     } elsif ($cdrom eq 'none') {
1277         return '';
1278     } elsif ($cdrom =~ m|^/|) {
1279         return $cdrom;
1280     } else {
1281         return PVE::Storage::path($storecfg, $cdrom);
1282     }
1283 }
1284
1285 # try to convert old style file names to volume IDs
1286 sub filename_to_volume_id {
1287     my ($vmid, $file, $media) = @_;
1288
1289      if (!($file eq 'none' || $file eq 'cdrom' ||
1290           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1291
1292         return if $file =~ m|/|;
1293
1294         if ($media && $media eq 'cdrom') {
1295             $file = "local:iso/$file";
1296         } else {
1297             $file = "local:$vmid/$file";
1298         }
1299     }
1300
1301     return $file;
1302 }
1303
1304 sub verify_media_type {
1305     my ($opt, $vtype, $media) = @_;
1306
1307     return if !$media;
1308
1309     my $etype;
1310     if ($media eq 'disk') {
1311         $etype = 'images';
1312     } elsif ($media eq 'cdrom') {
1313         $etype = 'iso';
1314     } else {
1315         die "internal error";
1316     }
1317
1318     return if ($vtype eq $etype);
1319
1320     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1321 }
1322
1323 sub cleanup_drive_path {
1324     my ($opt, $storecfg, $drive) = @_;
1325
1326     # try to convert filesystem paths to volume IDs
1327
1328     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1329         ($drive->{file} !~ m|^/dev/.+|) &&
1330         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1331         ($drive->{file} !~ m/^\d+$/)) {
1332         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1333         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1334             if !$vtype;
1335         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1336         verify_media_type($opt, $vtype, $drive->{media});
1337         $drive->{file} = $volid;
1338     }
1339
1340     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1341 }
1342
1343 sub parse_hotplug_features {
1344     my ($data) = @_;
1345
1346     my $res = {};
1347
1348     return $res if $data eq '0';
1349
1350     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1351
1352     foreach my $feature (PVE::Tools::split_list($data)) {
1353         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1354             $res->{$1} = 1;
1355         } else {
1356             die "invalid hotplug feature '$feature'\n";
1357         }
1358     }
1359     return $res;
1360 }
1361
1362 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1363 sub pve_verify_hotplug_features {
1364     my ($value, $noerr) = @_;
1365
1366     return $value if parse_hotplug_features($value);
1367
1368     return if $noerr;
1369
1370     die "unable to parse hotplug option\n";
1371 }
1372
1373 sub scsi_inquiry {
1374     my($fh, $noerr) = @_;
1375
1376     my $SG_IO = 0x2285;
1377     my $SG_GET_VERSION_NUM = 0x2282;
1378
1379     my $versionbuf = "\x00" x 8;
1380     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1381     if (!$ret) {
1382         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1383         return;
1384     }
1385     my $version = unpack("I", $versionbuf);
1386     if ($version < 30000) {
1387         die "scsi generic interface too old\n"  if !$noerr;
1388         return;
1389     }
1390
1391     my $buf = "\x00" x 36;
1392     my $sensebuf = "\x00" x 8;
1393     my $cmd = pack("C x3 C x1", 0x12, 36);
1394
1395     # see /usr/include/scsi/sg.h
1396     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1397
1398     my $packet = pack(
1399         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1400     );
1401
1402     $ret = ioctl($fh, $SG_IO, $packet);
1403     if (!$ret) {
1404         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1405         return;
1406     }
1407
1408     my @res = unpack($sg_io_hdr_t, $packet);
1409     if ($res[17] || $res[18]) {
1410         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my $res = {};
1415     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1416
1417     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1418     $res->{type} &= 0x1F;
1419
1420     return $res;
1421 }
1422
1423 sub path_is_scsi {
1424     my ($path) = @_;
1425
1426     my $fh = IO::File->new("+<$path") || return;
1427     my $res = scsi_inquiry($fh, 1);
1428     close($fh);
1429
1430     return $res;
1431 }
1432
1433 sub print_tabletdevice_full {
1434     my ($conf, $arch) = @_;
1435
1436     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1437
1438     # we use uhci for old VMs because tablet driver was buggy in older qemu
1439     my $usbbus;
1440     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1441         $usbbus = 'ehci';
1442     } else {
1443         $usbbus = 'uhci';
1444     }
1445
1446     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1447 }
1448
1449 sub print_keyboarddevice_full {
1450     my ($conf, $arch) = @_;
1451
1452     return if $arch ne 'aarch64';
1453
1454     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1455 }
1456
1457 my sub get_drive_id {
1458     my ($drive) = @_;
1459     return "$drive->{interface}$drive->{index}";
1460 }
1461
1462 sub print_drivedevice_full {
1463     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1464
1465     my $device = '';
1466     my $maxdev = 0;
1467
1468     my $drive_id = get_drive_id($drive);
1469     if ($drive->{interface} eq 'virtio') {
1470         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1471         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1472         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1473     } elsif ($drive->{interface} eq 'scsi') {
1474
1475         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1476         my $unit = $drive->{index} % $maxdev;
1477         my $devicetype = 'hd';
1478         my $path = '';
1479         if (drive_is_cdrom($drive)) {
1480             $devicetype = 'cd';
1481         } else {
1482             if ($drive->{file} =~ m|^/|) {
1483                 $path = $drive->{file};
1484                 if (my $info = path_is_scsi($path)) {
1485                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1486                         $devicetype = 'block';
1487                     } elsif ($info->{type} == 1) { # tape
1488                         $devicetype = 'generic';
1489                     }
1490                 }
1491             } else {
1492                  $path = PVE::Storage::path($storecfg, $drive->{file});
1493             }
1494
1495             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1496             my $version = extract_version($machine_type, kvm_user_version());
1497             if ($path =~ m/^iscsi\:\/\// &&
1498                !min_version($version, 4, 1)) {
1499                 $devicetype = 'generic';
1500             }
1501         }
1502
1503         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1504             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1505         } else {
1506             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1507                 .",lun=$drive->{index}";
1508         }
1509         $device .= ",drive=drive-$drive_id,id=$drive_id";
1510
1511         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1512             $device .= ",rotation_rate=1";
1513         }
1514         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1515
1516     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1517         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1518         my $controller = int($drive->{index} / $maxdev);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1521
1522         $device = "ide-$devicetype";
1523         if ($drive->{interface} eq 'ide') {
1524             $device .= ",bus=ide.$controller,unit=$unit";
1525         } else {
1526             $device .= ",bus=ahci$controller.$unit";
1527         }
1528         $device .= ",drive=drive-$drive_id,id=$drive_id";
1529
1530         if ($devicetype eq 'hd') {
1531             if (my $model = $drive->{model}) {
1532                 $model = URI::Escape::uri_unescape($model);
1533                 $device .= ",model=$model";
1534             }
1535             if ($drive->{ssd}) {
1536                 $device .= ",rotation_rate=1";
1537             }
1538         }
1539         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1540     } elsif ($drive->{interface} eq 'usb') {
1541         die "implement me";
1542         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1543     } else {
1544         die "unsupported interface type";
1545     }
1546
1547     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1548
1549     if (my $serial = $drive->{serial}) {
1550         $serial = URI::Escape::uri_unescape($serial);
1551         $device .= ",serial=$serial";
1552     }
1553
1554
1555     return $device;
1556 }
1557
1558 sub get_initiator_name {
1559     my $initiator;
1560
1561     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1562     while (defined(my $line = <$fh>)) {
1563         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1564         $initiator = $1;
1565         last;
1566     }
1567     $fh->close();
1568
1569     return $initiator;
1570 }
1571
1572 sub print_drive_commandline_full {
1573     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1574
1575     my $path;
1576     my $volid = $drive->{file};
1577     my $format = $drive->{format};
1578     my $drive_id = get_drive_id($drive);
1579
1580     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1581     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1582
1583     if (drive_is_cdrom($drive)) {
1584         $path = get_iso_path($storecfg, $vmid, $volid);
1585         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1586     } else {
1587         if ($storeid) {
1588             $path = PVE::Storage::path($storecfg, $volid);
1589             $format //= qemu_img_format($scfg, $volname);
1590         } else {
1591             $path = $volid;
1592             $format //= "raw";
1593         }
1594    }
1595
1596    my $is_rbd = $path =~ m/^rbd:/;
1597
1598     my $opts = '';
1599     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1600     foreach my $o (@qemu_drive_options) {
1601         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1602     }
1603
1604     # snapshot only accepts on|off
1605     if (defined($drive->{snapshot})) {
1606         my $v = $drive->{snapshot} ? 'on' : 'off';
1607         $opts .= ",snapshot=$v";
1608     }
1609
1610     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1611         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1612     }
1613
1614     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1615         my ($dir, $qmpname) = @$type;
1616         if (my $v = $drive->{"mbps$dir"}) {
1617             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1618         }
1619         if (my $v = $drive->{"mbps${dir}_max"}) {
1620             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1621         }
1622         if (my $v = $drive->{"bps${dir}_max_length"}) {
1623             $opts .= ",throttling.bps$qmpname-max-length=$v";
1624         }
1625         if (my $v = $drive->{"iops${dir}"}) {
1626             $opts .= ",throttling.iops$qmpname=$v";
1627         }
1628         if (my $v = $drive->{"iops${dir}_max"}) {
1629             $opts .= ",throttling.iops$qmpname-max=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}_max_length"}) {
1632             $opts .= ",throttling.iops$qmpname-max-length=$v";
1633         }
1634     }
1635
1636     if ($pbs_name) {
1637         $format = "rbd" if $is_rbd;
1638         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1639             if !$format;
1640         $opts .= ",format=alloc-track,file.driver=$format";
1641     } elsif ($format) {
1642         $opts .= ",format=$format";
1643     }
1644
1645     my $cache_direct = 0;
1646
1647     if (my $cache = $drive->{cache}) {
1648         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1649     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1650         $opts .= ",cache=none";
1651         $cache_direct = 1;
1652     }
1653
1654     # io_uring with cache mode writeback or writethrough on krbd will hang...
1655     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1656
1657     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1658     # sometimes, just plain disable...
1659     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1660
1661     if (!$drive->{aio}) {
1662         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1663             # io_uring supports all cache modes
1664             $opts .= ",aio=io_uring";
1665         } else {
1666             # aio native works only with O_DIRECT
1667             if($cache_direct) {
1668                 $opts .= ",aio=native";
1669             } else {
1670                 $opts .= ",aio=threads";
1671             }
1672         }
1673     }
1674
1675     if (!drive_is_cdrom($drive)) {
1676         my $detectzeroes;
1677         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1678             $detectzeroes = 'off';
1679         } elsif ($drive->{discard}) {
1680             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1681         } else {
1682             # This used to be our default with discard not being specified:
1683             $detectzeroes = 'on';
1684         }
1685
1686         # note: 'detect-zeroes' works per blockdev and we want it to persist
1687         # after the alloc-track is removed, so put it on 'file' directly
1688         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1689         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1690     }
1691
1692     if ($pbs_name) {
1693         $opts .= ",backing=$pbs_name";
1694         $opts .= ",auto-remove=on";
1695     }
1696
1697     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1698     my $file_param = "file";
1699     if ($pbs_name) {
1700         # non-rbd drivers require the underlying file to be a seperate block
1701         # node, so add a second .file indirection
1702         $file_param .= ".file" if !$is_rbd;
1703         $file_param .= ".filename";
1704     }
1705     my $pathinfo = $path ? "$file_param=$path," : '';
1706
1707     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1708 }
1709
1710 sub print_pbs_blockdev {
1711     my ($pbs_conf, $pbs_name) = @_;
1712     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1713     $blockdev .= ",repository=$pbs_conf->{repository}";
1714     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1715     $blockdev .= ",archive=$pbs_conf->{archive}";
1716     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1717     return $blockdev;
1718 }
1719
1720 sub print_netdevice_full {
1721     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1722
1723     my $device = $net->{model};
1724     if ($net->{model} eq 'virtio') {
1725          $device = 'virtio-net-pci';
1726      };
1727
1728     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1729     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1730     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1731         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1732         # and out of each queue plus one config interrupt and control vector queue
1733         my $vectors = $net->{queues} * 2 + 2;
1734         $tmpstr .= ",vectors=$vectors,mq=on";
1735     }
1736     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1737
1738     if (my $mtu = $net->{mtu}) {
1739         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1740             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1741             if ($mtu == 1) {
1742                  $mtu = $bridge_mtu;
1743             } elsif ($mtu < 576) {
1744                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1745             } elsif ($mtu > $bridge_mtu) {
1746                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1747             }
1748             $tmpstr .= ",host_mtu=$mtu";
1749         } else {
1750             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1751         }
1752     }
1753
1754     if ($use_old_bios_files) {
1755         my $romfile;
1756         if ($device eq 'virtio-net-pci') {
1757             $romfile = 'pxe-virtio.rom';
1758         } elsif ($device eq 'e1000') {
1759             $romfile = 'pxe-e1000.rom';
1760         } elsif ($device eq 'e1000e') {
1761             $romfile = 'pxe-e1000e.rom';
1762         } elsif ($device eq 'ne2k') {
1763             $romfile = 'pxe-ne2k_pci.rom';
1764         } elsif ($device eq 'pcnet') {
1765             $romfile = 'pxe-pcnet.rom';
1766         } elsif ($device eq 'rtl8139') {
1767             $romfile = 'pxe-rtl8139.rom';
1768         }
1769         $tmpstr .= ",romfile=$romfile" if $romfile;
1770     }
1771
1772     return $tmpstr;
1773 }
1774
1775 sub print_netdev_full {
1776     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1777
1778     my $i = '';
1779     if ($netid =~ m/^net(\d+)$/) {
1780         $i = int($1);
1781     }
1782
1783     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1784
1785     my $ifname = "tap${vmid}i$i";
1786
1787     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1788     die "interface name '$ifname' is too long (max 15 character)\n"
1789         if length($ifname) >= 16;
1790
1791     my $vhostparam = '';
1792     if (is_native($arch)) {
1793         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1794     }
1795
1796     my $vmname = $conf->{name} || "vm$vmid";
1797
1798     my $netdev = "";
1799     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1800
1801     if ($net->{bridge}) {
1802         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1803             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1804     } else {
1805         $netdev = "type=user,id=$netid,hostname=$vmname";
1806     }
1807
1808     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1809
1810     return $netdev;
1811 }
1812
1813 my $vga_map = {
1814     'cirrus' => 'cirrus-vga',
1815     'std' => 'VGA',
1816     'vmware' => 'vmware-svga',
1817     'virtio' => 'virtio-vga',
1818 };
1819
1820 sub print_vga_device {
1821     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1822
1823     my $type = $vga_map->{$vga->{type}};
1824     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1825         $type = 'virtio-gpu';
1826     }
1827     my $vgamem_mb = $vga->{memory};
1828
1829     my $max_outputs = '';
1830     if ($qxlnum) {
1831         $type = $id ? 'qxl' : 'qxl-vga';
1832
1833         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1834             # set max outputs so linux can have up to 4 qxl displays with one device
1835             if (min_version($machine_version, 4, 1)) {
1836                 $max_outputs = ",max_outputs=4";
1837             }
1838         }
1839     }
1840
1841     die "no devicetype for $vga->{type}\n" if !$type;
1842
1843     my $memory = "";
1844     if ($vgamem_mb) {
1845         if ($vga->{type} eq 'virtio') {
1846             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1847             $memory = ",max_hostmem=$bytes";
1848         } elsif ($qxlnum) {
1849             # from https://www.spice-space.org/multiple-monitors.html
1850             $memory = ",vgamem_mb=$vga->{memory}";
1851             my $ram = $vgamem_mb * 4;
1852             my $vram = $vgamem_mb * 2;
1853             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1854         } else {
1855             $memory = ",vgamem_mb=$vga->{memory}";
1856         }
1857     } elsif ($qxlnum && $id) {
1858         $memory = ",ram_size=67108864,vram_size=33554432";
1859     }
1860
1861     my $edidoff = "";
1862     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1863         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1864     }
1865
1866     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1867     my $vgaid = "vga" . ($id // '');
1868     my $pciaddr;
1869     if ($q35 && $vgaid eq 'vga') {
1870         # the first display uses pcie.0 bus on q35 machines
1871         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1872     } else {
1873         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1874     }
1875
1876     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1877 }
1878
1879 sub parse_number_sets {
1880     my ($set) = @_;
1881     my $res = [];
1882     foreach my $part (split(/;/, $set)) {
1883         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1884             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1885             push @$res, [ $1, $2 ];
1886         } else {
1887             die "invalid range: $part\n";
1888         }
1889     }
1890     return $res;
1891 }
1892
1893 sub parse_numa {
1894     my ($data) = @_;
1895
1896     my $res = parse_property_string($numa_fmt, $data);
1897     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1898     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1899     return $res;
1900 }
1901
1902 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1903 sub parse_net {
1904     my ($data) = @_;
1905
1906     my $res = eval { parse_property_string($net_fmt, $data) };
1907     if ($@) {
1908         warn $@;
1909         return;
1910     }
1911     if (!defined($res->{macaddr})) {
1912         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1913         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1914     }
1915     return $res;
1916 }
1917
1918 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1919 sub parse_ipconfig {
1920     my ($data) = @_;
1921
1922     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1923     if ($@) {
1924         warn $@;
1925         return;
1926     }
1927
1928     if ($res->{gw} && !$res->{ip}) {
1929         warn 'gateway specified without specifying an IP address';
1930         return;
1931     }
1932     if ($res->{gw6} && !$res->{ip6}) {
1933         warn 'IPv6 gateway specified without specifying an IPv6 address';
1934         return;
1935     }
1936     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1937         warn 'gateway specified together with DHCP';
1938         return;
1939     }
1940     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1941         # gw6 + auto/dhcp
1942         warn "IPv6 gateway specified together with $res->{ip6} address";
1943         return;
1944     }
1945
1946     if (!$res->{ip} && !$res->{ip6}) {
1947         return { ip => 'dhcp', ip6 => 'dhcp' };
1948     }
1949
1950     return $res;
1951 }
1952
1953 sub print_net {
1954     my $net = shift;
1955
1956     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1957 }
1958
1959 sub add_random_macs {
1960     my ($settings) = @_;
1961
1962     foreach my $opt (keys %$settings) {
1963         next if $opt !~ m/^net(\d+)$/;
1964         my $net = parse_net($settings->{$opt});
1965         next if !$net;
1966         $settings->{$opt} = print_net($net);
1967     }
1968 }
1969
1970 sub vm_is_volid_owner {
1971     my ($storecfg, $vmid, $volid) = @_;
1972
1973     if ($volid !~  m|^/|) {
1974         my ($path, $owner);
1975         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1976         if ($owner && ($owner == $vmid)) {
1977             return 1;
1978         }
1979     }
1980
1981     return;
1982 }
1983
1984 sub vmconfig_register_unused_drive {
1985     my ($storecfg, $vmid, $conf, $drive) = @_;
1986
1987     if (drive_is_cloudinit($drive)) {
1988         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1989         warn $@ if $@;
1990     } elsif (!drive_is_cdrom($drive)) {
1991         my $volid = $drive->{file};
1992         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1993             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1994         }
1995     }
1996 }
1997
1998 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1999 my $smbios1_fmt = {
2000     uuid => {
2001         type => 'string',
2002         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2003         format_description => 'UUID',
2004         description => "Set SMBIOS1 UUID.",
2005         optional => 1,
2006     },
2007     version => {
2008         type => 'string',
2009         pattern => '[A-Za-z0-9+\/]+={0,2}',
2010         format_description => 'Base64 encoded string',
2011         description => "Set SMBIOS1 version.",
2012         optional => 1,
2013     },
2014     serial => {
2015         type => 'string',
2016         pattern => '[A-Za-z0-9+\/]+={0,2}',
2017         format_description => 'Base64 encoded string',
2018         description => "Set SMBIOS1 serial number.",
2019         optional => 1,
2020     },
2021     manufacturer => {
2022         type => 'string',
2023         pattern => '[A-Za-z0-9+\/]+={0,2}',
2024         format_description => 'Base64 encoded string',
2025         description => "Set SMBIOS1 manufacturer.",
2026         optional => 1,
2027     },
2028     product => {
2029         type => 'string',
2030         pattern => '[A-Za-z0-9+\/]+={0,2}',
2031         format_description => 'Base64 encoded string',
2032         description => "Set SMBIOS1 product ID.",
2033         optional => 1,
2034     },
2035     sku => {
2036         type => 'string',
2037         pattern => '[A-Za-z0-9+\/]+={0,2}',
2038         format_description => 'Base64 encoded string',
2039         description => "Set SMBIOS1 SKU string.",
2040         optional => 1,
2041     },
2042     family => {
2043         type => 'string',
2044         pattern => '[A-Za-z0-9+\/]+={0,2}',
2045         format_description => 'Base64 encoded string',
2046         description => "Set SMBIOS1 family string.",
2047         optional => 1,
2048     },
2049     base64 => {
2050         type => 'boolean',
2051         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2052         optional => 1,
2053     },
2054 };
2055
2056 sub parse_smbios1 {
2057     my ($data) = @_;
2058
2059     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2060     warn $@ if $@;
2061     return $res;
2062 }
2063
2064 sub print_smbios1 {
2065     my ($smbios1) = @_;
2066     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2067 }
2068
2069 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2070
2071 sub parse_watchdog {
2072     my ($value) = @_;
2073
2074     return if !$value;
2075
2076     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2077     warn $@ if $@;
2078     return $res;
2079 }
2080
2081 sub parse_guest_agent {
2082     my ($conf) = @_;
2083
2084     return {} if !defined($conf->{agent});
2085
2086     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2087     warn $@ if $@;
2088
2089     # if the agent is disabled ignore the other potentially set properties
2090     return {} if !$res->{enabled};
2091     return $res;
2092 }
2093
2094 sub get_qga_key {
2095     my ($conf, $key) = @_;
2096     return undef if !defined($conf->{agent});
2097
2098     my $agent = parse_guest_agent($conf);
2099     return $agent->{$key};
2100 }
2101
2102 sub parse_vga {
2103     my ($value) = @_;
2104
2105     return {} if !$value;
2106     my $res = eval { parse_property_string($vga_fmt, $value) };
2107     warn $@ if $@;
2108     return $res;
2109 }
2110
2111 sub parse_rng {
2112     my ($value) = @_;
2113
2114     return if !$value;
2115
2116     my $res = eval { parse_property_string($rng_fmt, $value) };
2117     warn $@ if $@;
2118     return $res;
2119 }
2120
2121 sub parse_meta_info {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub new_meta_info_string {
2132     my () = @_; # for now do not allow to override any value
2133
2134     return PVE::JSONSchema::print_property_string(
2135         {
2136             'creation-qemu' => kvm_user_version(),
2137             ctime => "". int(time()),
2138         },
2139         $meta_info_fmt
2140     );
2141 }
2142
2143 sub qemu_created_version_fixups {
2144     my ($conf, $forcemachine, $kvmver) = @_;
2145
2146     my $meta = parse_meta_info($conf->{meta}) // {};
2147     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2148
2149     # check if we need to apply some handling for VMs that always use the latest machine version but
2150     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2151     # would be required (we do not want to pin machine version for non-windows OS type)
2152     if (
2153         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2154         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2155         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2156         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2157     ) {
2158         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2159         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2160             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2161             # and thus with the predictable interface naming of systemd
2162             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2163         }
2164     }
2165     return;
2166 }
2167
2168 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2169 sub verify_usb_device {
2170     my ($value, $noerr) = @_;
2171
2172     return $value if parse_usb_device($value);
2173
2174     return if $noerr;
2175
2176     die "unable to parse usb device\n";
2177 }
2178
2179 # add JSON properties for create and set function
2180 sub json_config_properties {
2181     my $prop = shift;
2182
2183     my $skip_json_config_opts = {
2184         parent => 1,
2185         snaptime => 1,
2186         vmstate => 1,
2187         runningmachine => 1,
2188         runningcpu => 1,
2189         meta => 1,
2190     };
2191
2192     foreach my $opt (keys %$confdesc) {
2193         next if $skip_json_config_opts->{$opt};
2194         $prop->{$opt} = $confdesc->{$opt};
2195     }
2196
2197     return $prop;
2198 }
2199
2200 # return copy of $confdesc_cloudinit to generate documentation
2201 sub cloudinit_config_properties {
2202
2203     return dclone($confdesc_cloudinit);
2204 }
2205
2206 sub check_type {
2207     my ($key, $value) = @_;
2208
2209     die "unknown setting '$key'\n" if !$confdesc->{$key};
2210
2211     my $type = $confdesc->{$key}->{type};
2212
2213     if (!defined($value)) {
2214         die "got undefined value\n";
2215     }
2216
2217     if ($value =~ m/[\n\r]/) {
2218         die "property contains a line feed\n";
2219     }
2220
2221     if ($type eq 'boolean') {
2222         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2223         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2224         die "type check ('boolean') failed - got '$value'\n";
2225     } elsif ($type eq 'integer') {
2226         return int($1) if $value =~ m/^(\d+)$/;
2227         die "type check ('integer') failed - got '$value'\n";
2228     } elsif ($type eq 'number') {
2229         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2230         die "type check ('number') failed - got '$value'\n";
2231     } elsif ($type eq 'string') {
2232         if (my $fmt = $confdesc->{$key}->{format}) {
2233             PVE::JSONSchema::check_format($fmt, $value);
2234             return $value;
2235         }
2236         $value =~ s/^\"(.*)\"$/$1/;
2237         return $value;
2238     } else {
2239         die "internal error"
2240     }
2241 }
2242
2243 sub destroy_vm {
2244     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2245
2246     my $conf = PVE::QemuConfig->load_config($vmid);
2247
2248     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2249
2250     if ($conf->{template}) {
2251         # check if any base image is still used by a linked clone
2252         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2253                 my ($ds, $drive) = @_;
2254                 return if drive_is_cdrom($drive);
2255
2256                 my $volid = $drive->{file};
2257                 return if !$volid || $volid =~ m|^/|;
2258
2259                 die "base volume '$volid' is still in use by linked cloned\n"
2260                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2261
2262         });
2263     }
2264
2265     my $volids = {};
2266     my $remove_owned_drive = sub {
2267         my ($ds, $drive) = @_;
2268         return if drive_is_cdrom($drive, 1);
2269
2270         my $volid = $drive->{file};
2271         return if !$volid || $volid =~ m|^/|;
2272         return if $volids->{$volid};
2273
2274         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2275         return if !$path || !$owner || ($owner != $vmid);
2276
2277         $volids->{$volid} = 1;
2278         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2279         warn "Could not remove disk '$volid', check manually: $@" if $@;
2280     };
2281
2282     # only remove disks owned by this VM (referenced in the config)
2283     my $include_opts = {
2284         include_unused => 1,
2285         extra_keys => ['vmstate'],
2286     };
2287     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2288
2289     for my $snap (values %{$conf->{snapshots}}) {
2290         next if !defined($snap->{vmstate});
2291         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2292         next if !defined($drive);
2293         $remove_owned_drive->('vmstate', $drive);
2294     }
2295
2296     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2297
2298     if ($purge_unreferenced) { # also remove unreferenced disk
2299         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2300         PVE::Storage::foreach_volid($vmdisks, sub {
2301             my ($volid, $sid, $volname, $d) = @_;
2302             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2303             warn $@ if $@;
2304         });
2305     }
2306
2307     if (defined $replacement_conf) {
2308         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2309     } else {
2310         PVE::QemuConfig->destroy_config($vmid);
2311     }
2312 }
2313
2314 sub parse_vm_config {
2315     my ($filename, $raw, $strict) = @_;
2316
2317     return if !defined($raw);
2318
2319     my $res = {
2320         digest => Digest::SHA::sha1_hex($raw),
2321         snapshots => {},
2322         pending => {},
2323     };
2324
2325     my $handle_error = sub {
2326         my ($msg) = @_;
2327
2328         if ($strict) {
2329             die $msg;
2330         } else {
2331             warn $msg;
2332         }
2333     };
2334
2335     $filename =~ m|/qemu-server/(\d+)\.conf$|
2336         || die "got strange filename '$filename'";
2337
2338     my $vmid = $1;
2339
2340     my $conf = $res;
2341     my $descr;
2342     my $section = '';
2343
2344     my @lines = split(/\n/, $raw);
2345     foreach my $line (@lines) {
2346         next if $line =~ m/^\s*$/;
2347
2348         if ($line =~ m/^\[PENDING\]\s*$/i) {
2349             $section = 'pending';
2350             if (defined($descr)) {
2351                 $descr =~ s/\s+$//;
2352                 $conf->{description} = $descr;
2353             }
2354             $descr = undef;
2355             $conf = $res->{$section} = {};
2356             next;
2357
2358         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2359             $section = $1;
2360             if (defined($descr)) {
2361                 $descr =~ s/\s+$//;
2362                 $conf->{description} = $descr;
2363             }
2364             $descr = undef;
2365             $conf = $res->{snapshots}->{$section} = {};
2366             next;
2367         }
2368
2369         if ($line =~ m/^\#(.*)\s*$/) {
2370             $descr = '' if !defined($descr);
2371             $descr .= PVE::Tools::decode_text($1) . "\n";
2372             next;
2373         }
2374
2375         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2376             $descr = '' if !defined($descr);
2377             $descr .= PVE::Tools::decode_text($2);
2378         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2379             $conf->{snapstate} = $1;
2380         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2381             my $key = $1;
2382             my $value = $2;
2383             $conf->{$key} = $value;
2384         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2385             my $value = $1;
2386             if ($section eq 'pending') {
2387                 $conf->{delete} = $value; # we parse this later
2388             } else {
2389                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2390             }
2391         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2392             my $key = $1;
2393             my $value = $2;
2394             eval { $value = check_type($key, $value); };
2395             if ($@) {
2396                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2397             } else {
2398                 $key = 'ide2' if $key eq 'cdrom';
2399                 my $fmt = $confdesc->{$key}->{format};
2400                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2401                     my $v = parse_drive($key, $value);
2402                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2403                         $v->{file} = $volid;
2404                         $value = print_drive($v);
2405                     } else {
2406                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2407                         next;
2408                     }
2409                 }
2410
2411                 $conf->{$key} = $value;
2412             }
2413         } else {
2414             $handle_error->("vm $vmid - unable to parse config: $line\n");
2415         }
2416     }
2417
2418     if (defined($descr)) {
2419         $descr =~ s/\s+$//;
2420         $conf->{description} = $descr;
2421     }
2422     delete $res->{snapstate}; # just to be sure
2423
2424     return $res;
2425 }
2426
2427 sub write_vm_config {
2428     my ($filename, $conf) = @_;
2429
2430     delete $conf->{snapstate}; # just to be sure
2431
2432     if ($conf->{cdrom}) {
2433         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2434         $conf->{ide2} = $conf->{cdrom};
2435         delete $conf->{cdrom};
2436     }
2437
2438     # we do not use 'smp' any longer
2439     if ($conf->{sockets}) {
2440         delete $conf->{smp};
2441     } elsif ($conf->{smp}) {
2442         $conf->{sockets} = $conf->{smp};
2443         delete $conf->{cores};
2444         delete $conf->{smp};
2445     }
2446
2447     my $used_volids = {};
2448
2449     my $cleanup_config = sub {
2450         my ($cref, $pending, $snapname) = @_;
2451
2452         foreach my $key (keys %$cref) {
2453             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2454                 $key eq 'snapstate' || $key eq 'pending';
2455             my $value = $cref->{$key};
2456             if ($key eq 'delete') {
2457                 die "propertry 'delete' is only allowed in [PENDING]\n"
2458                     if !$pending;
2459                 # fixme: check syntax?
2460                 next;
2461             }
2462             eval { $value = check_type($key, $value); };
2463             die "unable to parse value of '$key' - $@" if $@;
2464
2465             $cref->{$key} = $value;
2466
2467             if (!$snapname && is_valid_drivename($key)) {
2468                 my $drive = parse_drive($key, $value);
2469                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2470             }
2471         }
2472     };
2473
2474     &$cleanup_config($conf);
2475
2476     &$cleanup_config($conf->{pending}, 1);
2477
2478     foreach my $snapname (keys %{$conf->{snapshots}}) {
2479         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2480         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2481     }
2482
2483     # remove 'unusedX' settings if we re-add a volume
2484     foreach my $key (keys %$conf) {
2485         my $value = $conf->{$key};
2486         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2487             delete $conf->{$key};
2488         }
2489     }
2490
2491     my $generate_raw_config = sub {
2492         my ($conf, $pending) = @_;
2493
2494         my $raw = '';
2495
2496         # add description as comment to top of file
2497         if (defined(my $descr = $conf->{description})) {
2498             if ($descr) {
2499                 foreach my $cl (split(/\n/, $descr)) {
2500                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2501                 }
2502             } else {
2503                 $raw .= "#\n" if $pending;
2504             }
2505         }
2506
2507         foreach my $key (sort keys %$conf) {
2508             next if $key =~ /^(digest|description|pending|snapshots)$/;
2509             $raw .= "$key: $conf->{$key}\n";
2510         }
2511         return $raw;
2512     };
2513
2514     my $raw = &$generate_raw_config($conf);
2515
2516     if (scalar(keys %{$conf->{pending}})){
2517         $raw .= "\n[PENDING]\n";
2518         $raw .= &$generate_raw_config($conf->{pending}, 1);
2519     }
2520
2521     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2522         $raw .= "\n[$snapname]\n";
2523         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2524     }
2525
2526     return $raw;
2527 }
2528
2529 sub load_defaults {
2530
2531     my $res = {};
2532
2533     # we use static defaults from our JSON schema configuration
2534     foreach my $key (keys %$confdesc) {
2535         if (defined(my $default = $confdesc->{$key}->{default})) {
2536             $res->{$key} = $default;
2537         }
2538     }
2539
2540     return $res;
2541 }
2542
2543 sub config_list {
2544     my $vmlist = PVE::Cluster::get_vmlist();
2545     my $res = {};
2546     return $res if !$vmlist || !$vmlist->{ids};
2547     my $ids = $vmlist->{ids};
2548     my $nodename = nodename();
2549
2550     foreach my $vmid (keys %$ids) {
2551         my $d = $ids->{$vmid};
2552         next if !$d->{node} || $d->{node} ne $nodename;
2553         next if !$d->{type} || $d->{type} ne 'qemu';
2554         $res->{$vmid}->{exists} = 1;
2555     }
2556     return $res;
2557 }
2558
2559 # test if VM uses local resources (to prevent migration)
2560 sub check_local_resources {
2561     my ($conf, $noerr) = @_;
2562
2563     my @loc_res = ();
2564
2565     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2566     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2567
2568     push @loc_res, "ivshmem" if $conf->{ivshmem};
2569
2570     foreach my $k (keys %$conf) {
2571         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2572         # sockets are safe: they will recreated be on the target side post-migrate
2573         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2574         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2575     }
2576
2577     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2578
2579     return \@loc_res;
2580 }
2581
2582 # check if used storages are available on all nodes (use by migrate)
2583 sub check_storage_availability {
2584     my ($storecfg, $conf, $node) = @_;
2585
2586     PVE::QemuConfig->foreach_volume($conf, sub {
2587         my ($ds, $drive) = @_;
2588
2589         my $volid = $drive->{file};
2590         return if !$volid;
2591
2592         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2593         return if !$sid;
2594
2595         # check if storage is available on both nodes
2596         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2597         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2598
2599         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2600
2601         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2602             if !$scfg->{content}->{$vtype};
2603    });
2604 }
2605
2606 # list nodes where all VM images are available (used by has_feature API)
2607 sub shared_nodes {
2608     my ($conf, $storecfg) = @_;
2609
2610     my $nodelist = PVE::Cluster::get_nodelist();
2611     my $nodehash = { map { $_ => 1 } @$nodelist };
2612     my $nodename = nodename();
2613
2614     PVE::QemuConfig->foreach_volume($conf, sub {
2615         my ($ds, $drive) = @_;
2616
2617         my $volid = $drive->{file};
2618         return if !$volid;
2619
2620         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2621         if ($storeid) {
2622             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2623             if ($scfg->{disable}) {
2624                 $nodehash = {};
2625             } elsif (my $avail = $scfg->{nodes}) {
2626                 foreach my $node (keys %$nodehash) {
2627                     delete $nodehash->{$node} if !$avail->{$node};
2628                 }
2629             } elsif (!$scfg->{shared}) {
2630                 foreach my $node (keys %$nodehash) {
2631                     delete $nodehash->{$node} if $node ne $nodename
2632                 }
2633             }
2634         }
2635     });
2636
2637     return $nodehash
2638 }
2639
2640 sub check_local_storage_availability {
2641     my ($conf, $storecfg) = @_;
2642
2643     my $nodelist = PVE::Cluster::get_nodelist();
2644     my $nodehash = { map { $_ => {} } @$nodelist };
2645
2646     PVE::QemuConfig->foreach_volume($conf, sub {
2647         my ($ds, $drive) = @_;
2648
2649         my $volid = $drive->{file};
2650         return if !$volid;
2651
2652         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2653         if ($storeid) {
2654             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2655
2656             if ($scfg->{disable}) {
2657                 foreach my $node (keys %$nodehash) {
2658                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2659                 }
2660             } elsif (my $avail = $scfg->{nodes}) {
2661                 foreach my $node (keys %$nodehash) {
2662                     if (!$avail->{$node}) {
2663                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2664                     }
2665                 }
2666             }
2667         }
2668     });
2669
2670     foreach my $node (values %$nodehash) {
2671         if (my $unavail = $node->{unavailable_storages}) {
2672             $node->{unavailable_storages} = [ sort keys %$unavail ];
2673         }
2674     }
2675
2676     return $nodehash
2677 }
2678
2679 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2680 sub check_running {
2681     my ($vmid, $nocheck, $node) = @_;
2682
2683     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2684     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2685 }
2686
2687 sub vzlist {
2688
2689     my $vzlist = config_list();
2690
2691     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2692
2693     while (defined(my $de = $fd->read)) {
2694         next if $de !~ m/^(\d+)\.pid$/;
2695         my $vmid = $1;
2696         next if !defined($vzlist->{$vmid});
2697         if (my $pid = check_running($vmid)) {
2698             $vzlist->{$vmid}->{pid} = $pid;
2699         }
2700     }
2701
2702     return $vzlist;
2703 }
2704
2705 our $vmstatus_return_properties = {
2706     vmid => get_standard_option('pve-vmid'),
2707     status => {
2708         description => "Qemu process status.",
2709         type => 'string',
2710         enum => ['stopped', 'running'],
2711     },
2712     maxmem => {
2713         description => "Maximum memory in bytes.",
2714         type => 'integer',
2715         optional => 1,
2716         renderer => 'bytes',
2717     },
2718     maxdisk => {
2719         description => "Root disk size in bytes.",
2720         type => 'integer',
2721         optional => 1,
2722         renderer => 'bytes',
2723     },
2724     name => {
2725         description => "VM name.",
2726         type => 'string',
2727         optional => 1,
2728     },
2729     qmpstatus => {
2730         description => "Qemu QMP agent status.",
2731         type => 'string',
2732         optional => 1,
2733     },
2734     pid => {
2735         description => "PID of running qemu process.",
2736         type => 'integer',
2737         optional => 1,
2738     },
2739     uptime => {
2740         description => "Uptime.",
2741         type => 'integer',
2742         optional => 1,
2743         renderer => 'duration',
2744     },
2745     cpus => {
2746         description => "Maximum usable CPUs.",
2747         type => 'number',
2748         optional => 1,
2749     },
2750     lock => {
2751         description => "The current config lock, if any.",
2752         type => 'string',
2753         optional => 1,
2754     },
2755     tags => {
2756         description  => "The current configured tags, if any",
2757         type => 'string',
2758         optional => 1,
2759     },
2760     'running-machine' => {
2761         description => "The currently running machine type (if running).",
2762         type => 'string',
2763         optional => 1,
2764     },
2765     'running-qemu' => {
2766         description => "The currently running QEMU version (if running).",
2767         type => 'string',
2768         optional => 1,
2769     },
2770 };
2771
2772 my $last_proc_pid_stat;
2773
2774 # get VM status information
2775 # This must be fast and should not block ($full == false)
2776 # We only query KVM using QMP if $full == true (this can be slow)
2777 sub vmstatus {
2778     my ($opt_vmid, $full) = @_;
2779
2780     my $res = {};
2781
2782     my $storecfg = PVE::Storage::config();
2783
2784     my $list = vzlist();
2785     my $defaults = load_defaults();
2786
2787     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2788
2789     my $cpucount = $cpuinfo->{cpus} || 1;
2790
2791     foreach my $vmid (keys %$list) {
2792         next if $opt_vmid && ($vmid ne $opt_vmid);
2793
2794         my $conf = PVE::QemuConfig->load_config($vmid);
2795
2796         my $d = { vmid => int($vmid) };
2797         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2798
2799         # fixme: better status?
2800         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2801
2802         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2803         if (defined($size)) {
2804             $d->{disk} = 0; # no info available
2805             $d->{maxdisk} = $size;
2806         } else {
2807             $d->{disk} = 0;
2808             $d->{maxdisk} = 0;
2809         }
2810
2811         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2812             * ($conf->{cores} || $defaults->{cores});
2813         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2814         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2815
2816         $d->{name} = $conf->{name} || "VM $vmid";
2817         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2818             : $defaults->{memory}*(1024*1024);
2819
2820         if ($conf->{balloon}) {
2821             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2822             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2823                 : $defaults->{shares};
2824         }
2825
2826         $d->{uptime} = 0;
2827         $d->{cpu} = 0;
2828         $d->{mem} = 0;
2829
2830         $d->{netout} = 0;
2831         $d->{netin} = 0;
2832
2833         $d->{diskread} = 0;
2834         $d->{diskwrite} = 0;
2835
2836         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2837
2838         $d->{serial} = 1 if conf_has_serial($conf);
2839         $d->{lock} = $conf->{lock} if $conf->{lock};
2840         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2841
2842         $res->{$vmid} = $d;
2843     }
2844
2845     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2846     foreach my $dev (keys %$netdev) {
2847         next if $dev !~ m/^tap([1-9]\d*)i/;
2848         my $vmid = $1;
2849         my $d = $res->{$vmid};
2850         next if !$d;
2851
2852         $d->{netout} += $netdev->{$dev}->{receive};
2853         $d->{netin} += $netdev->{$dev}->{transmit};
2854
2855         if ($full) {
2856             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2857             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2858         }
2859
2860     }
2861
2862     my $ctime = gettimeofday;
2863
2864     foreach my $vmid (keys %$list) {
2865
2866         my $d = $res->{$vmid};
2867         my $pid = $d->{pid};
2868         next if !$pid;
2869
2870         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2871         next if !$pstat; # not running
2872
2873         my $used = $pstat->{utime} + $pstat->{stime};
2874
2875         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2876
2877         if ($pstat->{vsize}) {
2878             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2879         }
2880
2881         my $old = $last_proc_pid_stat->{$pid};
2882         if (!$old) {
2883             $last_proc_pid_stat->{$pid} = {
2884                 time => $ctime,
2885                 used => $used,
2886                 cpu => 0,
2887             };
2888             next;
2889         }
2890
2891         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2892
2893         if ($dtime > 1000) {
2894             my $dutime = $used -  $old->{used};
2895
2896             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2897             $last_proc_pid_stat->{$pid} = {
2898                 time => $ctime,
2899                 used => $used,
2900                 cpu => $d->{cpu},
2901             };
2902         } else {
2903             $d->{cpu} = $old->{cpu};
2904         }
2905     }
2906
2907     return $res if !$full;
2908
2909     my $qmpclient = PVE::QMPClient->new();
2910
2911     my $ballooncb = sub {
2912         my ($vmid, $resp) = @_;
2913
2914         my $info = $resp->{'return'};
2915         return if !$info->{max_mem};
2916
2917         my $d = $res->{$vmid};
2918
2919         # use memory assigned to VM
2920         $d->{maxmem} = $info->{max_mem};
2921         $d->{balloon} = $info->{actual};
2922
2923         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2924             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2925             $d->{freemem} = $info->{free_mem};
2926         }
2927
2928         $d->{ballooninfo} = $info;
2929     };
2930
2931     my $blockstatscb = sub {
2932         my ($vmid, $resp) = @_;
2933         my $data = $resp->{'return'} || [];
2934         my $totalrdbytes = 0;
2935         my $totalwrbytes = 0;
2936
2937         for my $blockstat (@$data) {
2938             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2939             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2940
2941             $blockstat->{device} =~ s/drive-//;
2942             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2943         }
2944         $res->{$vmid}->{diskread} = $totalrdbytes;
2945         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2946     };
2947
2948     my $machinecb = sub {
2949         my ($vmid, $resp) = @_;
2950         my $data = $resp->{'return'} || [];
2951
2952         $res->{$vmid}->{'running-machine'} =
2953             PVE::QemuServer::Machine::current_from_query_machines($data);
2954     };
2955
2956     my $versioncb = sub {
2957         my ($vmid, $resp) = @_;
2958         my $data = $resp->{'return'} // {};
2959         my $version = 'unknown';
2960
2961         if (my $v = $data->{qemu}) {
2962             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2963         }
2964
2965         $res->{$vmid}->{'running-qemu'} = $version;
2966     };
2967
2968     my $statuscb = sub {
2969         my ($vmid, $resp) = @_;
2970
2971         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2972         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2973         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2974         # this fails if ballon driver is not loaded, so this must be
2975         # the last commnand (following command are aborted if this fails).
2976         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2977
2978         my $status = 'unknown';
2979         if (!defined($status = $resp->{'return'}->{status})) {
2980             warn "unable to get VM status\n";
2981             return;
2982         }
2983
2984         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2985     };
2986
2987     foreach my $vmid (keys %$list) {
2988         next if $opt_vmid && ($vmid ne $opt_vmid);
2989         next if !$res->{$vmid}->{pid}; # not running
2990         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2991     }
2992
2993     $qmpclient->queue_execute(undef, 2);
2994
2995     foreach my $vmid (keys %$list) {
2996         next if $opt_vmid && ($vmid ne $opt_vmid);
2997         next if !$res->{$vmid}->{pid}; #not running
2998
2999         # we can't use the $qmpclient since it might have already aborted on
3000         # 'query-balloon', but this might also fail for older versions...
3001         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3002         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3003     }
3004
3005     foreach my $vmid (keys %$list) {
3006         next if $opt_vmid && ($vmid ne $opt_vmid);
3007         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3008     }
3009
3010     return $res;
3011 }
3012
3013 sub conf_has_serial {
3014     my ($conf) = @_;
3015
3016     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3017         if ($conf->{"serial$i"}) {
3018             return 1;
3019         }
3020     }
3021
3022     return 0;
3023 }
3024
3025 sub conf_has_audio {
3026     my ($conf, $id) = @_;
3027
3028     $id //= 0;
3029     my $audio = $conf->{"audio$id"};
3030     return if !defined($audio);
3031
3032     my $audioproperties = parse_property_string($audio_fmt, $audio);
3033     my $audiodriver = $audioproperties->{driver} // 'spice';
3034
3035     return {
3036         dev => $audioproperties->{device},
3037         dev_id => "audiodev$id",
3038         backend => $audiodriver,
3039         backend_id => "$audiodriver-backend${id}",
3040     };
3041 }
3042
3043 sub audio_devs {
3044     my ($audio, $audiopciaddr, $machine_version) = @_;
3045
3046     my $devs = [];
3047
3048     my $id = $audio->{dev_id};
3049     my $audiodev = "";
3050     if (min_version($machine_version, 4, 2)) {
3051         $audiodev = ",audiodev=$audio->{backend_id}";
3052     }
3053
3054     if ($audio->{dev} eq 'AC97') {
3055         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3056     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3057         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3058         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3059         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3060     } else {
3061         die "unkown audio device '$audio->{dev}', implement me!";
3062     }
3063
3064     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3065
3066     return $devs;
3067 }
3068
3069 sub get_tpm_paths {
3070     my ($vmid) = @_;
3071     return {
3072         socket => "/var/run/qemu-server/$vmid.swtpm",
3073         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3074     };
3075 }
3076
3077 sub add_tpm_device {
3078     my ($vmid, $devices, $conf) = @_;
3079
3080     return if !$conf->{tpmstate0};
3081
3082     my $paths = get_tpm_paths($vmid);
3083
3084     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3085     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3086     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3087 }
3088
3089 sub start_swtpm {
3090     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3091
3092     return if !$tpmdrive;
3093
3094     my $state;
3095     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3096     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3097     if ($storeid) {
3098         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3099     } else {
3100         $state = $tpm->{file};
3101     }
3102
3103     my $paths = get_tpm_paths($vmid);
3104
3105     # during migration, we will get state from remote
3106     #
3107     if (!$migration) {
3108         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3109         my $setup_cmd = [
3110             "swtpm_setup",
3111             "--tpmstate",
3112             "file://$state",
3113             "--createek",
3114             "--create-ek-cert",
3115             "--create-platform-cert",
3116             "--lock-nvram",
3117             "--config",
3118             "/etc/swtpm_setup.conf", # do not use XDG configs
3119             "--runas",
3120             "0", # force creation as root, error if not possible
3121             "--not-overwrite", # ignore existing state, do not modify
3122         ];
3123
3124         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3125         # TPM 2.0 supports ECC crypto, use if possible
3126         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3127
3128         run_command($setup_cmd, outfunc => sub {
3129             print "swtpm_setup: $1\n";
3130         });
3131     }
3132
3133     my $emulator_cmd = [
3134         "swtpm",
3135         "socket",
3136         "--tpmstate",
3137         "backend-uri=file://$state,mode=0600",
3138         "--ctrl",
3139         "type=unixio,path=$paths->{socket},mode=0600",
3140         "--pid",
3141         "file=$paths->{pid}",
3142         "--terminate", # terminate on QEMU disconnect
3143         "--daemon",
3144     ];
3145     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3146     run_command($emulator_cmd, outfunc => sub { print $1; });
3147
3148     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3149     while (! -e $paths->{pid}) {
3150         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3151         usleep(50_000);
3152     }
3153
3154     # return untainted PID of swtpm daemon so it can be killed on error
3155     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3156     return $1;
3157 }
3158
3159 sub vga_conf_has_spice {
3160     my ($vga) = @_;
3161
3162     my $vgaconf = parse_vga($vga);
3163     my $vgatype = $vgaconf->{type};
3164     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3165
3166     return $1 || 1;
3167 }
3168
3169 sub is_native($) {
3170     my ($arch) = @_;
3171     return get_host_arch() eq $arch;
3172 }
3173
3174 sub get_vm_arch {
3175     my ($conf) = @_;
3176     return $conf->{arch} // get_host_arch();
3177 }
3178
3179 my $default_machines = {
3180     x86_64 => 'pc',
3181     aarch64 => 'virt',
3182 };
3183
3184 sub get_installed_machine_version {
3185     my ($kvmversion) = @_;
3186     $kvmversion = kvm_user_version() if !defined($kvmversion);
3187     $kvmversion =~ m/^(\d+\.\d+)/;
3188     return $1;
3189 }
3190
3191 sub windows_get_pinned_machine_version {
3192     my ($machine, $base_version, $kvmversion) = @_;
3193
3194     my $pin_version = $base_version;
3195     if (!defined($base_version) ||
3196         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3197     ) {
3198         $pin_version = get_installed_machine_version($kvmversion);
3199     }
3200     if (!$machine || $machine eq 'pc') {
3201         $machine = "pc-i440fx-$pin_version";
3202     } elsif ($machine eq 'q35') {
3203         $machine = "pc-q35-$pin_version";
3204     } elsif ($machine eq 'virt') {
3205         $machine = "virt-$pin_version";
3206     } else {
3207         warn "unknown machine type '$machine', not touching that!\n";
3208     }
3209
3210     return $machine;
3211 }
3212
3213 sub get_vm_machine {
3214     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3215
3216     my $machine = $forcemachine || $conf->{machine};
3217
3218     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3219         $kvmversion //= kvm_user_version();
3220         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3221         # layout which confuses windows quite a bit and may result in various regressions..
3222         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3223         if (windows_version($conf->{ostype})) {
3224             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3225         }
3226         $arch //= 'x86_64';
3227         $machine ||= $default_machines->{$arch};
3228         if ($add_pve_version) {
3229             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3230             $machine .= "+pve$pvever";
3231         }
3232     }
3233
3234     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3235         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3236         $machine = $1 if $is_pxe;
3237
3238         # for version-pinned machines that do not include a pve-version (e.g.
3239         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3240         $machine .= '+pve0';
3241
3242         $machine .= '.pxe' if $is_pxe;
3243     }
3244
3245     return $machine;
3246 }
3247
3248 sub get_ovmf_files($$$) {
3249     my ($arch, $efidisk, $smm) = @_;
3250
3251     my $types = $OVMF->{$arch}
3252         or die "no OVMF images known for architecture '$arch'\n";
3253
3254     my $type = 'default';
3255     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3256         $type = $smm ? "4m" : "4m-no-smm";
3257         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3258     }
3259
3260     return $types->{$type}->@*;
3261 }
3262
3263 my $Arch2Qemu = {
3264     aarch64 => '/usr/bin/qemu-system-aarch64',
3265     x86_64 => '/usr/bin/qemu-system-x86_64',
3266 };
3267 sub get_command_for_arch($) {
3268     my ($arch) = @_;
3269     return '/usr/bin/kvm' if is_native($arch);
3270
3271     my $cmd = $Arch2Qemu->{$arch}
3272         or die "don't know how to emulate architecture '$arch'\n";
3273     return $cmd;
3274 }
3275
3276 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3277 # to use in a QEMU command line (-cpu element), first array_intersect the result
3278 # of query_supported_ with query_understood_. This is necessary because:
3279 #
3280 # a) query_understood_ returns flags the host cannot use and
3281 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3282 #    flags, but CPU settings - with most of them being flags. Those settings
3283 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3284 #
3285 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3286 # expensive. If you need the value returned from this, you can get it much
3287 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3288 # $accel being 'kvm' or 'tcg'.
3289 #
3290 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3291 # changes, automatically populating pmxcfs.
3292 #
3293 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3294 # since kvm and tcg machines support different flags
3295 #
3296 sub query_supported_cpu_flags {
3297     my ($arch) = @_;
3298
3299     $arch //= get_host_arch();
3300     my $default_machine = $default_machines->{$arch};
3301
3302     my $flags = {};
3303
3304     # FIXME: Once this is merged, the code below should work for ARM as well:
3305     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3306     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3307         $arch eq "aarch64";
3308
3309     my $kvm_supported = defined(kvm_version());
3310     my $qemu_cmd = get_command_for_arch($arch);
3311     my $fakevmid = -1;
3312     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3313
3314     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3315     my $query_supported_run_qemu = sub {
3316         my ($kvm) = @_;
3317
3318         my $flags = {};
3319         my $cmd = [
3320             $qemu_cmd,
3321             '-machine', $default_machine,
3322             '-display', 'none',
3323             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3324             '-mon', 'chardev=qmp,mode=control',
3325             '-pidfile', $pidfile,
3326             '-S', '-daemonize'
3327         ];
3328
3329         if (!$kvm) {
3330             push @$cmd, '-accel', 'tcg';
3331         }
3332
3333         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3334         die "QEMU flag querying VM exited with code " . $rc if $rc;
3335
3336         eval {
3337             my $cmd_result = mon_cmd(
3338                 $fakevmid,
3339                 'query-cpu-model-expansion',
3340                 type => 'full',
3341                 model => { name => 'host' }
3342             );
3343
3344             my $props = $cmd_result->{model}->{props};
3345             foreach my $prop (keys %$props) {
3346                 next if $props->{$prop} ne '1';
3347                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3348                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3349                 # We only keep those with underscores, to match /proc/cpuinfo
3350                 $prop =~ s/\.|-/_/g;
3351                 $flags->{$prop} = 1;
3352             }
3353         };
3354         my $err = $@;
3355
3356         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3357         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3358
3359         die $err if $err;
3360
3361         return [ sort keys %$flags ];
3362     };
3363
3364     # We need to query QEMU twice, since KVM and TCG have different supported flags
3365     PVE::QemuConfig->lock_config($fakevmid, sub {
3366         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3367         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3368
3369         if ($kvm_supported) {
3370             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3371             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3372         }
3373     });
3374
3375     return $flags;
3376 }
3377
3378 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3379 my $understood_cpu_flag_dir = "/usr/share/kvm";
3380 sub query_understood_cpu_flags {
3381     my $arch = get_host_arch();
3382     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3383
3384     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3385         if ! -e $filepath;
3386
3387     my $raw = file_get_contents($filepath);
3388     $raw =~ s/^\s+|\s+$//g;
3389     my @flags = split(/\s+/, $raw);
3390
3391     return \@flags;
3392 }
3393
3394 my sub get_cpuunits {
3395     my ($conf) = @_;
3396     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3397 }
3398
3399 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3400 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3401 my sub should_disable_smm {
3402     my ($conf, $vga) = @_;
3403
3404     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3405         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3406 }
3407
3408 sub config_to_command {
3409     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3410         $pbs_backing) = @_;
3411
3412     my $cmd = [];
3413     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3414     my $devices = [];
3415     my $bridges = {};
3416     my $ostype = $conf->{ostype};
3417     my $winversion = windows_version($ostype);
3418     my $kvm = $conf->{kvm};
3419     my $nodename = nodename();
3420
3421     my $arch = get_vm_arch($conf);
3422     my $kvm_binary = get_command_for_arch($arch);
3423     my $kvmver = kvm_user_version($kvm_binary);
3424
3425     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3426         $kvmver //= "undefined";
3427         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3428     }
3429
3430     my $add_pve_version = min_version($kvmver, 4, 1);
3431
3432     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3433     my $machine_version = extract_version($machine_type, $kvmver);
3434     $kvm //= 1 if is_native($arch);
3435
3436     $machine_version =~ m/(\d+)\.(\d+)/;
3437     my ($machine_major, $machine_minor) = ($1, $2);
3438
3439     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3440         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3441     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3442         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3443             ." please upgrade node '$nodename'\n"
3444     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3445         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3446         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3447             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3448             ." node '$nodename'\n";
3449     }
3450
3451     # if a specific +pve version is required for a feature, use $version_guard
3452     # instead of min_version to allow machines to be run with the minimum
3453     # required version
3454     my $required_pve_version = 0;
3455     my $version_guard = sub {
3456         my ($major, $minor, $pve) = @_;
3457         return 0 if !min_version($machine_version, $major, $minor, $pve);
3458         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3459         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3460         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3461         return 1;
3462     };
3463
3464     if ($kvm && !defined kvm_version()) {
3465         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3466             ." or enable in BIOS.\n";
3467     }
3468
3469     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3470     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3471     my $use_old_bios_files = undef;
3472     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3473
3474     my $cpuunits = get_cpuunits($conf);
3475
3476     push @$cmd, $kvm_binary;
3477
3478     push @$cmd, '-id', $vmid;
3479
3480     my $vmname = $conf->{name} || "vm$vmid";
3481
3482     push @$cmd, '-name', $vmname;
3483
3484     push @$cmd, '-no-shutdown';
3485
3486     my $use_virtio = 0;
3487
3488     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3489     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3490     push @$cmd, '-mon', "chardev=qmp,mode=control";
3491
3492     if (min_version($machine_version, 2, 12)) {
3493         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3494         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3495     }
3496
3497     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3498
3499     push @$cmd, '-daemonize';
3500
3501     if ($conf->{smbios1}) {
3502         my $smbios_conf = parse_smbios1($conf->{smbios1});
3503         if ($smbios_conf->{base64}) {
3504             # Do not pass base64 flag to qemu
3505             delete $smbios_conf->{base64};
3506             my $smbios_string = "";
3507             foreach my $key (keys %$smbios_conf) {
3508                 my $value;
3509                 if ($key eq "uuid") {
3510                     $value = $smbios_conf->{uuid}
3511                 } else {
3512                     $value = decode_base64($smbios_conf->{$key});
3513                 }
3514                 # qemu accepts any binary data, only commas need escaping by double comma
3515                 $value =~ s/,/,,/g;
3516                 $smbios_string .= "," . $key . "=" . $value if $value;
3517             }
3518             push @$cmd, '-smbios', "type=1" . $smbios_string;
3519         } else {
3520             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3521         }
3522     }
3523
3524     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3525         my $d;
3526         if (my $efidisk = $conf->{efidisk0}) {
3527             $d = parse_drive('efidisk0', $efidisk);
3528         }
3529
3530         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3531         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3532
3533         my ($path, $format);
3534         my $read_only_str = '';
3535         if ($d) {
3536             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3537             $format = $d->{format};
3538             if ($storeid) {
3539                 $path = PVE::Storage::path($storecfg, $d->{file});
3540                 if (!defined($format)) {
3541                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3542                     $format = qemu_img_format($scfg, $volname);
3543                 }
3544             } else {
3545                 $path = $d->{file};
3546                 die "efidisk format must be specified\n"
3547                     if !defined($format);
3548             }
3549
3550             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3551         } else {
3552             warn "no efidisk configured! Using temporary efivars disk.\n";
3553             $path = "/tmp/$vmid-ovmf.fd";
3554             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3555             $format = 'raw';
3556         }
3557
3558         my $size_str = "";
3559
3560         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3561             $size_str = ",size=" . (-s $ovmf_vars);
3562         }
3563
3564         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3565         my $cache = "";
3566         if ($path =~ m/^rbd:/) {
3567                 $cache = ',cache=writeback';
3568                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3569         }
3570
3571         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3572         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3573     }
3574
3575     if ($q35) { # tell QEMU to load q35 config early
3576         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3577         if (min_version($machine_version, 4, 0)) {
3578             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3579         } else {
3580             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3581         }
3582     }
3583
3584     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3585         push @$cmd, $fixups->@*;
3586     }
3587
3588     if ($conf->{vmgenid}) {
3589         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3590     }
3591
3592     # add usb controllers
3593     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3594         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3595     push @$devices, @usbcontrollers if @usbcontrollers;
3596     my $vga = parse_vga($conf->{vga});
3597
3598     my $qxlnum = vga_conf_has_spice($conf->{vga});
3599     $vga->{type} = 'qxl' if $qxlnum;
3600
3601     if (!$vga->{type}) {
3602         if ($arch eq 'aarch64') {
3603             $vga->{type} = 'virtio';
3604         } elsif (min_version($machine_version, 2, 9)) {
3605             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3606         } else {
3607             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3608         }
3609     }
3610
3611     # enable absolute mouse coordinates (needed by vnc)
3612     my $tablet = $conf->{tablet};
3613     if (!defined($tablet)) {
3614         $tablet = $defaults->{tablet};
3615         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3616         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3617     }
3618
3619     if ($tablet) {
3620         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3621         my $kbd = print_keyboarddevice_full($conf, $arch);
3622         push @$devices, '-device', $kbd if defined($kbd);
3623     }
3624
3625     my $bootorder = device_bootorder($conf);
3626
3627     # host pci device passthrough
3628     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3629         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3630
3631     # usb devices
3632     my $usb_dev_features = {};
3633     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3634
3635     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3636         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3637     push @$devices, @usbdevices if @usbdevices;
3638
3639     # serial devices
3640     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3641         my $path = $conf->{"serial$i"} or next;
3642         if ($path eq 'socket') {
3643             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3644             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3645             # On aarch64, serial0 is the UART device. Qemu only allows
3646             # connecting UART devices via the '-serial' command line, as
3647             # the device has a fixed slot on the hardware...
3648             if ($arch eq 'aarch64' && $i == 0) {
3649                 push @$devices, '-serial', "chardev:serial$i";
3650             } else {
3651                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3652             }
3653         } else {
3654             die "no such serial device\n" if ! -c $path;
3655             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3656             push @$devices, '-device', "isa-serial,chardev=serial$i";
3657         }
3658     }
3659
3660     # parallel devices
3661     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3662         if (my $path = $conf->{"parallel$i"}) {
3663             die "no such parallel device\n" if ! -c $path;
3664             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3665             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3666             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3667         }
3668     }
3669
3670     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3671         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3672         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3673         push @$devices, @$audio_devs;
3674     }
3675
3676     add_tpm_device($vmid, $devices, $conf);
3677
3678     my $sockets = 1;
3679     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3680     $sockets = $conf->{sockets} if  $conf->{sockets};
3681
3682     my $cores = $conf->{cores} || 1;
3683
3684     my $maxcpus = $sockets * $cores;
3685
3686     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3687
3688     my $allowed_vcpus = $cpuinfo->{cpus};
3689
3690     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3691
3692     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3693         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3694         for (my $i = 2; $i <= $vcpus; $i++)  {
3695             my $cpustr = print_cpu_device($conf,$i);
3696             push @$cmd, '-device', $cpustr;
3697         }
3698
3699     } else {
3700
3701         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3702     }
3703     push @$cmd, '-nodefaults';
3704
3705     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3706
3707     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3708
3709     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3710
3711     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3712         push @$devices, '-device', print_vga_device(
3713             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3714         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3715         push @$cmd,  '-vnc', "unix:$socket,password=on";
3716     } else {
3717         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3718         push @$cmd, '-nographic';
3719     }
3720
3721     # time drift fix
3722     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3723     my $useLocaltime = $conf->{localtime};
3724
3725     if ($winversion >= 5) { # windows
3726         $useLocaltime = 1 if !defined($conf->{localtime});
3727
3728         # use time drift fix when acpi is enabled
3729         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3730             $tdf = 1 if !defined($conf->{tdf});
3731         }
3732     }
3733
3734     if ($winversion >= 6) {
3735         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3736         push @$cmd, '-no-hpet';
3737     }
3738
3739     push @$rtcFlags, 'driftfix=slew' if $tdf;
3740
3741     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3742         push @$rtcFlags, "base=$conf->{startdate}";
3743     } elsif ($useLocaltime) {
3744         push @$rtcFlags, 'base=localtime';
3745     }
3746
3747     if ($forcecpu) {
3748         push @$cmd, '-cpu', $forcecpu;
3749     } else {
3750         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3751     }
3752
3753     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3754
3755     push @$cmd, '-S' if $conf->{freeze};
3756
3757     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3758
3759     my $guest_agent = parse_guest_agent($conf);
3760
3761     if ($guest_agent->{enabled}) {
3762         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3763         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3764
3765         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3766             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3767             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3768             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3769         } elsif ($guest_agent->{type} eq 'isa') {
3770             push @$devices, '-device', "isa-serial,chardev=qga0";
3771         }
3772     }
3773
3774     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3775     if ($rng && $version_guard->(4, 1, 2)) {
3776         check_rng_source($rng->{source});
3777
3778         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3779         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3780         my $limiter_str = "";
3781         if ($max_bytes) {
3782             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3783         }
3784
3785         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3786         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3787         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3788     }
3789
3790     my $spice_port;
3791
3792     if ($qxlnum) {
3793         if ($qxlnum > 1) {
3794             if ($winversion){
3795                 for (my $i = 1; $i < $qxlnum; $i++){
3796                     push @$devices, '-device', print_vga_device(
3797                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3798                 }
3799             } else {
3800                 # assume other OS works like Linux
3801                 my ($ram, $vram) = ("134217728", "67108864");
3802                 if ($vga->{memory}) {
3803                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3804                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3805                 }
3806                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3807                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3808             }
3809         }
3810
3811         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3812
3813         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3814         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3815         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3816
3817         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3818         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3819         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3820
3821         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3822         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3823
3824         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3825         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3826         if ($spice_enhancement->{foldersharing}) {
3827             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3828             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3829         }
3830
3831         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3832         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3833             if $spice_enhancement->{videostreaming};
3834
3835         push @$devices, '-spice', "$spice_opts";
3836     }
3837
3838     # enable balloon by default, unless explicitly disabled
3839     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3840         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3841         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3842     }
3843
3844     if ($conf->{watchdog}) {
3845         my $wdopts = parse_watchdog($conf->{watchdog});
3846         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3847         my $watchdog = $wdopts->{model} || 'i6300esb';
3848         push @$devices, '-device', "$watchdog$pciaddr";
3849         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3850     }
3851
3852     my $vollist = [];
3853     my $scsicontroller = {};
3854     my $ahcicontroller = {};
3855     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3856
3857     # Add iscsi initiator name if available
3858     if (my $initiator = get_initiator_name()) {
3859         push @$devices, '-iscsi', "initiator-name=$initiator";
3860     }
3861
3862     PVE::QemuConfig->foreach_volume($conf, sub {
3863         my ($ds, $drive) = @_;
3864
3865         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3866             check_volume_storage_type($storecfg, $drive->{file});
3867             push @$vollist, $drive->{file};
3868         }
3869
3870         # ignore efidisk here, already added in bios/fw handling code above
3871         return if $drive->{interface} eq 'efidisk';
3872         # similar for TPM
3873         return if $drive->{interface} eq 'tpmstate';
3874
3875         $use_virtio = 1 if $ds =~ m/^virtio/;
3876
3877         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3878
3879         if ($drive->{interface} eq 'virtio'){
3880            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3881         }
3882
3883         if ($drive->{interface} eq 'scsi') {
3884
3885             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3886
3887             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3888                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3889
3890             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3891             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3892
3893             my $iothread = '';
3894             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3895                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3896                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3897             } elsif ($drive->{iothread}) {
3898                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3899             }
3900
3901             my $queues = '';
3902             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3903                 $queues = ",num_queues=$drive->{queues}";
3904             }
3905
3906             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3907                 if !$scsicontroller->{$controller};
3908             $scsicontroller->{$controller}=1;
3909         }
3910
3911         if ($drive->{interface} eq 'sata') {
3912             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3913             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3914             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3915                 if !$ahcicontroller->{$controller};
3916             $ahcicontroller->{$controller}=1;
3917         }
3918
3919         my $pbs_conf = $pbs_backing->{$ds};
3920         my $pbs_name = undef;
3921         if ($pbs_conf) {
3922             $pbs_name = "drive-$ds-pbs";
3923             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3924         }
3925
3926         my $drive_cmd = print_drive_commandline_full(
3927             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3928
3929         # extra protection for templates, but SATA and IDE don't support it..
3930         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3931
3932         push @$devices, '-drive',$drive_cmd;
3933         push @$devices, '-device', print_drivedevice_full(
3934             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3935     });
3936
3937     for (my $i = 0; $i < $MAX_NETS; $i++) {
3938         my $netname = "net$i";
3939
3940         next if !$conf->{$netname};
3941         my $d = parse_net($conf->{$netname});
3942         next if !$d;
3943
3944         $use_virtio = 1 if $d->{model} eq 'virtio';
3945
3946         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3947
3948         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3949         push @$devices, '-netdev', $netdevfull;
3950
3951         my $netdevicefull = print_netdevice_full(
3952             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3953
3954         push @$devices, '-device', $netdevicefull;
3955     }
3956
3957     if ($conf->{ivshmem}) {
3958         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3959
3960         my $bus;
3961         if ($q35) {
3962             $bus = print_pcie_addr("ivshmem");
3963         } else {
3964             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3965         }
3966
3967         my $ivshmem_name = $ivshmem->{name} // $vmid;
3968         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3969
3970         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3971         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3972             .",size=$ivshmem->{size}M";
3973     }
3974
3975     # pci.4 is nested in pci.1
3976     $bridges->{1} = 1 if $bridges->{4};
3977
3978     if (!$q35) { # add pci bridges
3979         if (min_version($machine_version, 2, 3)) {
3980            $bridges->{1} = 1;
3981            $bridges->{2} = 1;
3982         }
3983         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3984     }
3985
3986     for my $k (sort {$b cmp $a} keys %$bridges) {
3987         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3988
3989         my $k_name = $k;
3990         if ($k == 2 && $legacy_igd) {
3991             $k_name = "$k-igd";
3992         }
3993         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3994         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3995
3996         if ($q35) { # add after -readconfig pve-q35.cfg
3997             splice @$devices, 2, 0, '-device', $devstr;
3998         } else {
3999             unshift @$devices, '-device', $devstr if $k > 0;
4000         }
4001     }
4002
4003     if (!$kvm) {
4004         push @$machineFlags, 'accel=tcg';
4005     }
4006
4007     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4008
4009     my $machine_type_min = $machine_type;
4010     if ($add_pve_version) {
4011         $machine_type_min =~ s/\+pve\d+$//;
4012         $machine_type_min .= "+pve$required_pve_version";
4013     }
4014     push @$machineFlags, "type=${machine_type_min}";
4015
4016     push @$cmd, @$devices;
4017     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4018     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4019     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4020
4021     if (my $vmstate = $conf->{vmstate}) {
4022         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4023         push @$vollist, $vmstate;
4024         push @$cmd, '-loadstate', $statepath;
4025         print "activating and using '$vmstate' as vmstate\n";
4026     }
4027
4028     if (PVE::QemuConfig->is_template($conf)) {
4029         # needed to workaround base volumes being read-only
4030         push @$cmd, '-snapshot';
4031     }
4032
4033     # add custom args
4034     if ($conf->{args}) {
4035         my $aa = PVE::Tools::split_args($conf->{args});
4036         push @$cmd, @$aa;
4037     }
4038
4039     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4040 }
4041
4042 sub check_rng_source {
4043     my ($source) = @_;
4044
4045     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4046     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4047         if ! -e $source;
4048
4049     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4050     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4051         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4052         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4053         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4054             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4055             ." to the host.\n";
4056     }
4057 }
4058
4059 sub spice_port {
4060     my ($vmid) = @_;
4061
4062     my $res = mon_cmd($vmid, 'query-spice');
4063
4064     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4065 }
4066
4067 sub vm_devices_list {
4068     my ($vmid) = @_;
4069
4070     my $res = mon_cmd($vmid, 'query-pci');
4071     my $devices_to_check = [];
4072     my $devices = {};
4073     foreach my $pcibus (@$res) {
4074         push @$devices_to_check, @{$pcibus->{devices}},
4075     }
4076
4077     while (@$devices_to_check) {
4078         my $to_check = [];
4079         for my $d (@$devices_to_check) {
4080             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4081             next if !$d->{'pci_bridge'};
4082
4083             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4084             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4085         }
4086         $devices_to_check = $to_check;
4087     }
4088
4089     my $resblock = mon_cmd($vmid, 'query-block');
4090     foreach my $block (@$resblock) {
4091         if($block->{device} =~ m/^drive-(\S+)/){
4092                 $devices->{$1} = 1;
4093         }
4094     }
4095
4096     my $resmice = mon_cmd($vmid, 'query-mice');
4097     foreach my $mice (@$resmice) {
4098         if ($mice->{name} eq 'QEMU HID Tablet') {
4099             $devices->{tablet} = 1;
4100             last;
4101         }
4102     }
4103
4104     # for usb devices there is no query-usb
4105     # but we can iterate over the entries in
4106     # qom-list path=/machine/peripheral
4107     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4108     foreach my $per (@$resperipheral) {
4109         if ($per->{name} =~ m/^usb\d+$/) {
4110             $devices->{$per->{name}} = 1;
4111         }
4112     }
4113
4114     return $devices;
4115 }
4116
4117 sub vm_deviceplug {
4118     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4119
4120     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4121
4122     my $devices_list = vm_devices_list($vmid);
4123     return 1 if defined($devices_list->{$deviceid});
4124
4125     # add PCI bridge if we need it for the device
4126     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4127
4128     if ($deviceid eq 'tablet') {
4129         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4130     } elsif ($deviceid eq 'keyboard') {
4131         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4132     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4133         die "usb hotplug currently not reliable\n";
4134         # since we can't reliably hot unplug all added usb devices and usb
4135         # passthrough breaks live migration we disable usb hotplugging for now
4136         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4137     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4138         qemu_iothread_add($vmid, $deviceid, $device);
4139
4140         qemu_driveadd($storecfg, $vmid, $device);
4141         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4142
4143         qemu_deviceadd($vmid, $devicefull);
4144         eval { qemu_deviceaddverify($vmid, $deviceid); };
4145         if (my $err = $@) {
4146             eval { qemu_drivedel($vmid, $deviceid); };
4147             warn $@ if $@;
4148             die $err;
4149         }
4150     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4151         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4152         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4153         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4154
4155         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4156
4157         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4158             qemu_iothread_add($vmid, $deviceid, $device);
4159             $devicefull .= ",iothread=iothread-$deviceid";
4160         }
4161
4162         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4163             $devicefull .= ",num_queues=$device->{queues}";
4164         }
4165
4166         qemu_deviceadd($vmid, $devicefull);
4167         qemu_deviceaddverify($vmid, $deviceid);
4168     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4169         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4170         qemu_driveadd($storecfg, $vmid, $device);
4171
4172         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4173         eval { qemu_deviceadd($vmid, $devicefull); };
4174         if (my $err = $@) {
4175             eval { qemu_drivedel($vmid, $deviceid); };
4176             warn $@ if $@;
4177             die $err;
4178         }
4179     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4180         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4181
4182         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4183         my $use_old_bios_files = undef;
4184         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4185
4186         my $netdevicefull = print_netdevice_full(
4187             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4188         qemu_deviceadd($vmid, $netdevicefull);
4189         eval {
4190             qemu_deviceaddverify($vmid, $deviceid);
4191             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4192         };
4193         if (my $err = $@) {
4194             eval { qemu_netdevdel($vmid, $deviceid); };
4195             warn $@ if $@;
4196             die $err;
4197         }
4198     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4199         my $bridgeid = $2;
4200         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4201         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4202
4203         qemu_deviceadd($vmid, $devicefull);
4204         qemu_deviceaddverify($vmid, $deviceid);
4205     } else {
4206         die "can't hotplug device '$deviceid'\n";
4207     }
4208
4209     return 1;
4210 }
4211
4212 # fixme: this should raise exceptions on error!
4213 sub vm_deviceunplug {
4214     my ($vmid, $conf, $deviceid) = @_;
4215
4216     my $devices_list = vm_devices_list($vmid);
4217     return 1 if !defined($devices_list->{$deviceid});
4218
4219     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4220     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4221
4222     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4223         qemu_devicedel($vmid, $deviceid);
4224     } elsif ($deviceid =~ m/^usb\d+$/) {
4225         die "usb hotplug currently not reliable\n";
4226         # when unplugging usb devices this way, there may be remaining usb
4227         # controllers/hubs so we disable it for now
4228         #qemu_devicedel($vmid, $deviceid);
4229         #qemu_devicedelverify($vmid, $deviceid);
4230     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4231         my $device = parse_drive($deviceid, $conf->{$deviceid});
4232
4233         qemu_devicedel($vmid, $deviceid);
4234         qemu_devicedelverify($vmid, $deviceid);
4235         qemu_drivedel($vmid, $deviceid);
4236         qemu_iothread_del($vmid, $deviceid, $device);
4237     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4238         qemu_devicedel($vmid, $deviceid);
4239         qemu_devicedelverify($vmid, $deviceid);
4240     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4241         my $device = parse_drive($deviceid, $conf->{$deviceid});
4242
4243         qemu_devicedel($vmid, $deviceid);
4244         qemu_drivedel($vmid, $deviceid);
4245         qemu_deletescsihw($conf, $vmid, $deviceid);
4246
4247         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4248             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4249     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4250         qemu_devicedel($vmid, $deviceid);
4251         qemu_devicedelverify($vmid, $deviceid);
4252         qemu_netdevdel($vmid, $deviceid);
4253     } else {
4254         die "can't unplug device '$deviceid'\n";
4255     }
4256
4257     return 1;
4258 }
4259
4260 sub qemu_deviceadd {
4261     my ($vmid, $devicefull) = @_;
4262
4263     $devicefull = "driver=".$devicefull;
4264     my %options =  split(/[=,]/, $devicefull);
4265
4266     mon_cmd($vmid, "device_add" , %options);
4267 }
4268
4269 sub qemu_devicedel {
4270     my ($vmid, $deviceid) = @_;
4271
4272     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4273 }
4274
4275 sub qemu_iothread_add {
4276     my ($vmid, $deviceid, $device) = @_;
4277
4278     if ($device->{iothread}) {
4279         my $iothreads = vm_iothreads_list($vmid);
4280         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4281     }
4282 }
4283
4284 sub qemu_iothread_del {
4285     my ($vmid, $deviceid, $device) = @_;
4286
4287     if ($device->{iothread}) {
4288         my $iothreads = vm_iothreads_list($vmid);
4289         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4290     }
4291 }
4292
4293 sub qemu_objectadd {
4294     my ($vmid, $objectid, $qomtype) = @_;
4295
4296     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4297
4298     return 1;
4299 }
4300
4301 sub qemu_objectdel {
4302     my ($vmid, $objectid) = @_;
4303
4304     mon_cmd($vmid, "object-del", id => $objectid);
4305
4306     return 1;
4307 }
4308
4309 sub qemu_driveadd {
4310     my ($storecfg, $vmid, $device) = @_;
4311
4312     my $kvmver = get_running_qemu_version($vmid);
4313     my $io_uring = min_version($kvmver, 6, 0);
4314     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4315     $drive =~ s/\\/\\\\/g;
4316     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4317
4318     # If the command succeeds qemu prints: "OK"
4319     return 1 if $ret =~ m/OK/s;
4320
4321     die "adding drive failed: $ret\n";
4322 }
4323
4324 sub qemu_drivedel {
4325     my ($vmid, $deviceid) = @_;
4326
4327     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4328     $ret =~ s/^\s+//;
4329
4330     return 1 if $ret eq "";
4331
4332     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4333     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4334
4335     die "deleting drive $deviceid failed : $ret\n";
4336 }
4337
4338 sub qemu_deviceaddverify {
4339     my ($vmid, $deviceid) = @_;
4340
4341     for (my $i = 0; $i <= 5; $i++) {
4342          my $devices_list = vm_devices_list($vmid);
4343          return 1 if defined($devices_list->{$deviceid});
4344          sleep 1;
4345     }
4346
4347     die "error on hotplug device '$deviceid'\n";
4348 }
4349
4350
4351 sub qemu_devicedelverify {
4352     my ($vmid, $deviceid) = @_;
4353
4354     # need to verify that the device is correctly removed as device_del
4355     # is async and empty return is not reliable
4356
4357     for (my $i = 0; $i <= 5; $i++) {
4358          my $devices_list = vm_devices_list($vmid);
4359          return 1 if !defined($devices_list->{$deviceid});
4360          sleep 1;
4361     }
4362
4363     die "error on hot-unplugging device '$deviceid'\n";
4364 }
4365
4366 sub qemu_findorcreatescsihw {
4367     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4368
4369     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4370
4371     my $scsihwid="$controller_prefix$controller";
4372     my $devices_list = vm_devices_list($vmid);
4373
4374     if (!defined($devices_list->{$scsihwid})) {
4375         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4376     }
4377
4378     return 1;
4379 }
4380
4381 sub qemu_deletescsihw {
4382     my ($conf, $vmid, $opt) = @_;
4383
4384     my $device = parse_drive($opt, $conf->{$opt});
4385
4386     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4387         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4388         return 1;
4389     }
4390
4391     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4392
4393     my $devices_list = vm_devices_list($vmid);
4394     foreach my $opt (keys %{$devices_list}) {
4395         if (is_valid_drivename($opt)) {
4396             my $drive = parse_drive($opt, $conf->{$opt});
4397             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4398                 return 1;
4399             }
4400         }
4401     }
4402
4403     my $scsihwid="scsihw$controller";
4404
4405     vm_deviceunplug($vmid, $conf, $scsihwid);
4406
4407     return 1;
4408 }
4409
4410 sub qemu_add_pci_bridge {
4411     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4412
4413     my $bridges = {};
4414
4415     my $bridgeid;
4416
4417     print_pci_addr($device, $bridges, $arch, $machine_type);
4418
4419     while (my ($k, $v) = each %$bridges) {
4420         $bridgeid = $k;
4421     }
4422     return 1 if !defined($bridgeid) || $bridgeid < 1;
4423
4424     my $bridge = "pci.$bridgeid";
4425     my $devices_list = vm_devices_list($vmid);
4426
4427     if (!defined($devices_list->{$bridge})) {
4428         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4429     }
4430
4431     return 1;
4432 }
4433
4434 sub qemu_set_link_status {
4435     my ($vmid, $device, $up) = @_;
4436
4437     mon_cmd($vmid, "set_link", name => $device,
4438                up => $up ? JSON::true : JSON::false);
4439 }
4440
4441 sub qemu_netdevadd {
4442     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4443
4444     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4445     my %options =  split(/[=,]/, $netdev);
4446
4447     if (defined(my $vhost = $options{vhost})) {
4448         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4449     }
4450
4451     if (defined(my $queues = $options{queues})) {
4452         $options{queues} = $queues + 0;
4453     }
4454
4455     mon_cmd($vmid, "netdev_add",  %options);
4456     return 1;
4457 }
4458
4459 sub qemu_netdevdel {
4460     my ($vmid, $deviceid) = @_;
4461
4462     mon_cmd($vmid, "netdev_del", id => $deviceid);
4463 }
4464
4465 sub qemu_usb_hotplug {
4466     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4467
4468     return if !$device;
4469
4470     # remove the old one first
4471     vm_deviceunplug($vmid, $conf, $deviceid);
4472
4473     # check if xhci controller is necessary and available
4474     if ($device->{usb3}) {
4475
4476         my $devicelist = vm_devices_list($vmid);
4477
4478         if (!$devicelist->{xhci}) {
4479             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4480             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4481         }
4482     }
4483     my $d = parse_usb_device($device->{host});
4484     $d->{usb3} = $device->{usb3};
4485
4486     # add the new one
4487     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4488 }
4489
4490 sub qemu_cpu_hotplug {
4491     my ($vmid, $conf, $vcpus) = @_;
4492
4493     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4494
4495     my $sockets = 1;
4496     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4497     $sockets = $conf->{sockets} if  $conf->{sockets};
4498     my $cores = $conf->{cores} || 1;
4499     my $maxcpus = $sockets * $cores;
4500
4501     $vcpus = $maxcpus if !$vcpus;
4502
4503     die "you can't add more vcpus than maxcpus\n"
4504         if $vcpus > $maxcpus;
4505
4506     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4507
4508     if ($vcpus < $currentvcpus) {
4509
4510         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4511
4512             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4513                 qemu_devicedel($vmid, "cpu$i");
4514                 my $retry = 0;
4515                 my $currentrunningvcpus = undef;
4516                 while (1) {
4517                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4518                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4519                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4520                     $retry++;
4521                     sleep 1;
4522                 }
4523                 #update conf after each succesfull cpu unplug
4524                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4525                 PVE::QemuConfig->write_config($vmid, $conf);
4526             }
4527         } else {
4528             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4529         }
4530
4531         return;
4532     }
4533
4534     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4535     die "vcpus in running vm does not match its configuration\n"
4536         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4537
4538     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4539
4540         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4541             my $cpustr = print_cpu_device($conf, $i);
4542             qemu_deviceadd($vmid, $cpustr);
4543
4544             my $retry = 0;
4545             my $currentrunningvcpus = undef;
4546             while (1) {
4547                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4548                 last if scalar(@{$currentrunningvcpus}) == $i;
4549                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4550                 sleep 1;
4551                 $retry++;
4552             }
4553             #update conf after each succesfull cpu hotplug
4554             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4555             PVE::QemuConfig->write_config($vmid, $conf);
4556         }
4557     } else {
4558
4559         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4560             mon_cmd($vmid, "cpu-add", id => int($i));
4561         }
4562     }
4563 }
4564
4565 sub qemu_block_set_io_throttle {
4566     my ($vmid, $deviceid,
4567         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4568         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4569         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4570         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4571
4572     return if !check_running($vmid) ;
4573
4574     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4575         bps => int($bps),
4576         bps_rd => int($bps_rd),
4577         bps_wr => int($bps_wr),
4578         iops => int($iops),
4579         iops_rd => int($iops_rd),
4580         iops_wr => int($iops_wr),
4581         bps_max => int($bps_max),
4582         bps_rd_max => int($bps_rd_max),
4583         bps_wr_max => int($bps_wr_max),
4584         iops_max => int($iops_max),
4585         iops_rd_max => int($iops_rd_max),
4586         iops_wr_max => int($iops_wr_max),
4587         bps_max_length => int($bps_max_length),
4588         bps_rd_max_length => int($bps_rd_max_length),
4589         bps_wr_max_length => int($bps_wr_max_length),
4590         iops_max_length => int($iops_max_length),
4591         iops_rd_max_length => int($iops_rd_max_length),
4592         iops_wr_max_length => int($iops_wr_max_length),
4593     );
4594
4595 }
4596
4597 sub qemu_block_resize {
4598     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4599
4600     my $running = check_running($vmid);
4601
4602     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4603
4604     return if !$running;
4605
4606     my $padding = (1024 - $size % 1024) % 1024;
4607     $size = $size + $padding;
4608
4609     mon_cmd(
4610         $vmid,
4611         "block_resize",
4612         device => $deviceid,
4613         size => int($size),
4614         timeout => 60,
4615     );
4616 }
4617
4618 sub qemu_volume_snapshot {
4619     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4620
4621     my $running = check_running($vmid);
4622
4623     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4624         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4625     } else {
4626         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4627     }
4628 }
4629
4630 sub qemu_volume_snapshot_delete {
4631     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4632
4633     my $running = check_running($vmid);
4634
4635     if($running) {
4636
4637         $running = undef;
4638         my $conf = PVE::QemuConfig->load_config($vmid);
4639         PVE::QemuConfig->foreach_volume($conf, sub {
4640             my ($ds, $drive) = @_;
4641             $running = 1 if $drive->{file} eq $volid;
4642         });
4643     }
4644
4645     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4646         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4647     } else {
4648         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4649     }
4650 }
4651
4652 sub set_migration_caps {
4653     my ($vmid, $savevm) = @_;
4654
4655     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4656
4657     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4658     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4659
4660     my $cap_ref = [];
4661
4662     my $enabled_cap = {
4663         "auto-converge" => 1,
4664         "xbzrle" => 1,
4665         "x-rdma-pin-all" => 0,
4666         "zero-blocks" => 0,
4667         "compress" => 0,
4668         "dirty-bitmaps" => $dirty_bitmaps,
4669     };
4670
4671     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4672
4673     for my $supported_capability (@$supported_capabilities) {
4674         push @$cap_ref, {
4675             capability => $supported_capability->{capability},
4676             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4677         };
4678     }
4679
4680     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4681 }
4682
4683 sub foreach_volid {
4684     my ($conf, $func, @param) = @_;
4685
4686     my $volhash = {};
4687
4688     my $test_volid = sub {
4689         my ($key, $drive, $snapname) = @_;
4690
4691         my $volid = $drive->{file};
4692         return if !$volid;
4693
4694         $volhash->{$volid}->{cdrom} //= 1;
4695         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4696
4697         my $replicate = $drive->{replicate} // 1;
4698         $volhash->{$volid}->{replicate} //= 0;
4699         $volhash->{$volid}->{replicate} = 1 if $replicate;
4700
4701         $volhash->{$volid}->{shared} //= 0;
4702         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4703
4704         $volhash->{$volid}->{referenced_in_config} //= 0;
4705         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4706
4707         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4708             if defined($snapname);
4709
4710         my $size = $drive->{size};
4711         $volhash->{$volid}->{size} //= $size if $size;
4712
4713         $volhash->{$volid}->{is_vmstate} //= 0;
4714         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4715
4716         $volhash->{$volid}->{is_tpmstate} //= 0;
4717         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4718
4719         $volhash->{$volid}->{is_unused} //= 0;
4720         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4721
4722         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4723     };
4724
4725     my $include_opts = {
4726         extra_keys => ['vmstate'],
4727         include_unused => 1,
4728     };
4729
4730     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4731     foreach my $snapname (keys %{$conf->{snapshots}}) {
4732         my $snap = $conf->{snapshots}->{$snapname};
4733         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4734     }
4735
4736     foreach my $volid (keys %$volhash) {
4737         &$func($volid, $volhash->{$volid}, @param);
4738     }
4739 }
4740
4741 my $fast_plug_option = {
4742     'lock' => 1,
4743     'name' => 1,
4744     'onboot' => 1,
4745     'shares' => 1,
4746     'startup' => 1,
4747     'description' => 1,
4748     'protection' => 1,
4749     'vmstatestorage' => 1,
4750     'hookscript' => 1,
4751     'tags' => 1,
4752 };
4753
4754 # hotplug changes in [PENDING]
4755 # $selection hash can be used to only apply specified options, for
4756 # example: { cores => 1 } (only apply changed 'cores')
4757 # $errors ref is used to return error messages
4758 sub vmconfig_hotplug_pending {
4759     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4760
4761     my $defaults = load_defaults();
4762     my $arch = get_vm_arch($conf);
4763     my $machine_type = get_vm_machine($conf, undef, $arch);
4764
4765     # commit values which do not have any impact on running VM first
4766     # Note: those option cannot raise errors, we we do not care about
4767     # $selection and always apply them.
4768
4769     my $add_error = sub {
4770         my ($opt, $msg) = @_;
4771         $errors->{$opt} = "hotplug problem - $msg";
4772     };
4773
4774     my $changes = 0;
4775     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4776         if ($fast_plug_option->{$opt}) {
4777             $conf->{$opt} = $conf->{pending}->{$opt};
4778             delete $conf->{pending}->{$opt};
4779             $changes = 1;
4780         }
4781     }
4782
4783     if ($changes) {
4784         PVE::QemuConfig->write_config($vmid, $conf);
4785     }
4786
4787     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4788
4789     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4790     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4791     foreach my $opt (sort keys %$pending_delete_hash) {
4792         next if $selection && !$selection->{$opt};
4793         my $force = $pending_delete_hash->{$opt}->{force};
4794         eval {
4795             if ($opt eq 'hotplug') {
4796                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4797             } elsif ($opt eq 'tablet') {
4798                 die "skip\n" if !$hotplug_features->{usb};
4799                 if ($defaults->{tablet}) {
4800                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4801                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4802                         if $arch eq 'aarch64';
4803                 } else {
4804                     vm_deviceunplug($vmid, $conf, 'tablet');
4805                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4806                 }
4807             } elsif ($opt =~ m/^usb\d+/) {
4808                 die "skip\n";
4809                 # since we cannot reliably hot unplug usb devices we are disabling it
4810                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4811                 #vm_deviceunplug($vmid, $conf, $opt);
4812             } elsif ($opt eq 'vcpus') {
4813                 die "skip\n" if !$hotplug_features->{cpu};
4814                 qemu_cpu_hotplug($vmid, $conf, undef);
4815             } elsif ($opt eq 'balloon') {
4816                 # enable balloon device is not hotpluggable
4817                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4818                 # here we reset the ballooning value to memory
4819                 my $balloon = $conf->{memory} || $defaults->{memory};
4820                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4821             } elsif ($fast_plug_option->{$opt}) {
4822                 # do nothing
4823             } elsif ($opt =~ m/^net(\d+)$/) {
4824                 die "skip\n" if !$hotplug_features->{network};
4825                 vm_deviceunplug($vmid, $conf, $opt);
4826             } elsif (is_valid_drivename($opt)) {
4827                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4828                 vm_deviceunplug($vmid, $conf, $opt);
4829                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4830             } elsif ($opt =~ m/^memory$/) {
4831                 die "skip\n" if !$hotplug_features->{memory};
4832                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4833             } elsif ($opt eq 'cpuunits') {
4834                 $cgroup->change_cpu_shares(undef, 1024);
4835             } elsif ($opt eq 'cpulimit') {
4836                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4837             } else {
4838                 die "skip\n";
4839             }
4840         };
4841         if (my $err = $@) {
4842             &$add_error($opt, $err) if $err ne "skip\n";
4843         } else {
4844             delete $conf->{$opt};
4845             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4846         }
4847     }
4848
4849     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4850     $apply_pending_cloudinit = sub {
4851         return if $apply_pending_cloudinit_done; # once is enough
4852         $apply_pending_cloudinit_done = 1; # once is enough
4853
4854         my ($key, $value) = @_;
4855
4856         my @cloudinit_opts = keys %$confdesc_cloudinit;
4857         foreach my $opt (keys %{$conf->{pending}}) {
4858             next if !grep { $_ eq $opt } @cloudinit_opts;
4859             $conf->{$opt} = delete $conf->{pending}->{$opt};
4860         }
4861
4862         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4863         foreach my $opt (sort keys %$pending_delete_hash) {
4864             next if !grep { $_ eq $opt } @cloudinit_opts;
4865             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4866             delete $conf->{$opt};
4867         }
4868
4869         my $new_conf = { %$conf };
4870         $new_conf->{$key} = $value;
4871         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4872     };
4873
4874     foreach my $opt (keys %{$conf->{pending}}) {
4875         next if $selection && !$selection->{$opt};
4876         my $value = $conf->{pending}->{$opt};
4877         eval {
4878             if ($opt eq 'hotplug') {
4879                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4880             } elsif ($opt eq 'tablet') {
4881                 die "skip\n" if !$hotplug_features->{usb};
4882                 if ($value == 1) {
4883                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4884                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4885                         if $arch eq 'aarch64';
4886                 } elsif ($value == 0) {
4887                     vm_deviceunplug($vmid, $conf, 'tablet');
4888                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4889                 }
4890             } elsif ($opt =~ m/^usb\d+$/) {
4891                 die "skip\n";
4892                 # since we cannot reliably hot unplug usb devices we disable it for now
4893                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4894                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4895                 #die "skip\n" if !$d;
4896                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4897             } elsif ($opt eq 'vcpus') {
4898                 die "skip\n" if !$hotplug_features->{cpu};
4899                 qemu_cpu_hotplug($vmid, $conf, $value);
4900             } elsif ($opt eq 'balloon') {
4901                 # enable/disable balloning device is not hotpluggable
4902                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4903                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4904                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4905
4906                 # allow manual ballooning if shares is set to zero
4907                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4908                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4909                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4910                 }
4911             } elsif ($opt =~ m/^net(\d+)$/) {
4912                 # some changes can be done without hotplug
4913                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4914                                     $vmid, $opt, $value, $arch, $machine_type);
4915             } elsif (is_valid_drivename($opt)) {
4916                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4917                 # some changes can be done without hotplug
4918                 my $drive = parse_drive($opt, $value);
4919                 if (drive_is_cloudinit($drive)) {
4920                     &$apply_pending_cloudinit($opt, $value);
4921                 }
4922                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4923                                      $vmid, $opt, $value, $arch, $machine_type);
4924             } elsif ($opt =~ m/^memory$/) { #dimms
4925                 die "skip\n" if !$hotplug_features->{memory};
4926                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4927             } elsif ($opt eq 'cpuunits') {
4928                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4929             } elsif ($opt eq 'cpulimit') {
4930                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4931                 $cgroup->change_cpu_quota($cpulimit, 100000);
4932             } elsif ($opt eq 'agent') {
4933                 vmconfig_update_agent($conf, $opt, $value);
4934             } else {
4935                 die "skip\n";  # skip non-hot-pluggable options
4936             }
4937         };
4938         if (my $err = $@) {
4939             &$add_error($opt, $err) if $err ne "skip\n";
4940         } else {
4941             $conf->{$opt} = $value;
4942             delete $conf->{pending}->{$opt};
4943         }
4944     }
4945
4946     PVE::QemuConfig->write_config($vmid, $conf);
4947 }
4948
4949 sub try_deallocate_drive {
4950     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4951
4952     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4953         my $volid = $drive->{file};
4954         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4955             my $sid = PVE::Storage::parse_volume_id($volid);
4956             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4957
4958             # check if the disk is really unused
4959             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4960                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4961             PVE::Storage::vdisk_free($storecfg, $volid);
4962             return 1;
4963         } else {
4964             # If vm is not owner of this disk remove from config
4965             return 1;
4966         }
4967     }
4968
4969     return;
4970 }
4971
4972 sub vmconfig_delete_or_detach_drive {
4973     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4974
4975     my $drive = parse_drive($opt, $conf->{$opt});
4976
4977     my $rpcenv = PVE::RPCEnvironment::get();
4978     my $authuser = $rpcenv->get_user();
4979
4980     if ($force) {
4981         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4982         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4983     } else {
4984         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4985     }
4986 }
4987
4988
4989
4990 sub vmconfig_apply_pending {
4991     my ($vmid, $conf, $storecfg, $errors) = @_;
4992
4993     return if !scalar(keys %{$conf->{pending}});
4994
4995     my $add_apply_error = sub {
4996         my ($opt, $msg) = @_;
4997         my $err_msg = "unable to apply pending change $opt : $msg";
4998         $errors->{$opt} = $err_msg;
4999         warn $err_msg;
5000     };
5001
5002     # cold plug
5003
5004     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5005     foreach my $opt (sort keys %$pending_delete_hash) {
5006         my $force = $pending_delete_hash->{$opt}->{force};
5007         eval {
5008             if ($opt =~ m/^unused/) {
5009                 die "internal error";
5010             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5011                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5012             }
5013         };
5014         if (my $err = $@) {
5015             $add_apply_error->($opt, $err);
5016         } else {
5017             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5018             delete $conf->{$opt};
5019         }
5020     }
5021
5022     PVE::QemuConfig->cleanup_pending($conf);
5023
5024     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5025         next if $opt eq 'delete'; # just to be sure
5026         eval {
5027             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5028                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5029             }
5030         };
5031         if (my $err = $@) {
5032             $add_apply_error->($opt, $err);
5033         } else {
5034             $conf->{$opt} = delete $conf->{pending}->{$opt};
5035         }
5036     }
5037
5038     # write all changes at once to avoid unnecessary i/o
5039     PVE::QemuConfig->write_config($vmid, $conf);
5040 }
5041
5042 sub vmconfig_update_net {
5043     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5044
5045     my $newnet = parse_net($value);
5046
5047     if ($conf->{$opt}) {
5048         my $oldnet = parse_net($conf->{$opt});
5049
5050         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5051             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5052             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5053             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5054
5055             # for non online change, we try to hot-unplug
5056             die "skip\n" if !$hotplug;
5057             vm_deviceunplug($vmid, $conf, $opt);
5058         } else {
5059
5060             die "internal error" if $opt !~ m/net(\d+)/;
5061             my $iface = "tap${vmid}i$1";
5062
5063             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5064                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5065                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5066                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5067                 PVE::Network::tap_unplug($iface);
5068
5069                 if ($have_sdn) {
5070                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5071                 } else {
5072                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5073                 }
5074             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5075                 # Rate can be applied on its own but any change above needs to
5076                 # include the rate in tap_plug since OVS resets everything.
5077                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5078             }
5079
5080             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5081                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5082             }
5083
5084             return 1;
5085         }
5086     }
5087
5088     if ($hotplug) {
5089         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5090     } else {
5091         die "skip\n";
5092     }
5093 }
5094
5095 sub vmconfig_update_agent {
5096     my ($conf, $opt, $value) = @_;
5097
5098     die "skip\n" if !$conf->{$opt};
5099
5100     my $hotplug_options = { fstrim_cloned_disks => 1 };
5101
5102     my $old_agent = parse_guest_agent($conf);
5103     my $agent = parse_guest_agent({$opt => $value});
5104
5105     for my $option (keys %$agent) { # added/changed options
5106         next if defined($hotplug_options->{$option});
5107         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5108     }
5109
5110     for my $option (keys %$old_agent) { # removed options
5111         next if defined($hotplug_options->{$option});
5112         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5113     }
5114
5115     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5116 }
5117
5118 sub vmconfig_update_disk {
5119     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5120
5121     my $drive = parse_drive($opt, $value);
5122
5123     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5124         my $media = $drive->{media} || 'disk';
5125         my $oldmedia = $old_drive->{media} || 'disk';
5126         die "unable to change media type\n" if $media ne $oldmedia;
5127
5128         if (!drive_is_cdrom($old_drive)) {
5129
5130             if ($drive->{file} ne $old_drive->{file}) {
5131
5132                 die "skip\n" if !$hotplug;
5133
5134                 # unplug and register as unused
5135                 vm_deviceunplug($vmid, $conf, $opt);
5136                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5137
5138             } else {
5139                 # update existing disk
5140
5141                 # skip non hotpluggable value
5142                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5143                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5144                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5145                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5146                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5147                     die "skip\n";
5148                 }
5149
5150                 # apply throttle
5151                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5152                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5153                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5154                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5155                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5156                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5157                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5158                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5159                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5160                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5161                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5162                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5163                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5164                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5165                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5166                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5167                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5168                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5169
5170                     qemu_block_set_io_throttle(
5171                         $vmid,"drive-$opt",
5172                         ($drive->{mbps} || 0)*1024*1024,
5173                         ($drive->{mbps_rd} || 0)*1024*1024,
5174                         ($drive->{mbps_wr} || 0)*1024*1024,
5175                          $drive->{iops} || 0,
5176                          $drive->{iops_rd} || 0,
5177                          $drive->{iops_wr} || 0,
5178                         ($drive->{mbps_max} || 0)*1024*1024,
5179                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5180                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5181                          $drive->{iops_max} || 0,
5182                          $drive->{iops_rd_max} || 0,
5183                          $drive->{iops_wr_max} || 0,
5184                          $drive->{bps_max_length} || 1,
5185                          $drive->{bps_rd_max_length} || 1,
5186                          $drive->{bps_wr_max_length} || 1,
5187                          $drive->{iops_max_length} || 1,
5188                          $drive->{iops_rd_max_length} || 1,
5189                          $drive->{iops_wr_max_length} || 1,
5190                     );
5191
5192                 }
5193
5194                 return 1;
5195             }
5196
5197         } else { # cdrom
5198
5199             if ($drive->{file} eq 'none') {
5200                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5201                 if (drive_is_cloudinit($old_drive)) {
5202                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5203                 }
5204             } else {
5205                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5206
5207                 # force eject if locked
5208                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5209
5210                 if ($path) {
5211                     mon_cmd($vmid, "blockdev-change-medium",
5212                         id => "$opt", filename => "$path");
5213                 }
5214             }
5215
5216             return 1;
5217         }
5218     }
5219
5220     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5221     # hotplug new disks
5222     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5223     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5224 }
5225
5226 # called in locked context by incoming migration
5227 sub vm_migrate_get_nbd_disks {
5228     my ($storecfg, $conf, $replicated_volumes) = @_;
5229
5230     my $local_volumes = {};
5231     PVE::QemuConfig->foreach_volume($conf, sub {
5232         my ($ds, $drive) = @_;
5233
5234         return if drive_is_cdrom($drive);
5235         return if $ds eq 'tpmstate0';
5236
5237         my $volid = $drive->{file};
5238
5239         return if !$volid;
5240
5241         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5242
5243         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5244         return if $scfg->{shared};
5245
5246         # replicated disks re-use existing state via bitmap
5247         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5248         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5249     });
5250     return $local_volumes;
5251 }
5252
5253 # called in locked context by incoming migration
5254 sub vm_migrate_alloc_nbd_disks {
5255     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5256
5257     my $format = undef;
5258
5259     my $nbd = {};
5260     foreach my $opt (sort keys %$source_volumes) {
5261         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5262
5263         if ($use_existing) {
5264             $nbd->{$opt}->{drivestr} = print_drive($drive);
5265             $nbd->{$opt}->{volid} = $volid;
5266             $nbd->{$opt}->{replicated} = 1;
5267             next;
5268         }
5269
5270         # If a remote storage is specified and the format of the original
5271         # volume is not available there, fall back to the default format.
5272         # Otherwise use the same format as the original.
5273         if (!$storagemap->{identity}) {
5274             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5275             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5276             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5277             my $fileFormat = qemu_img_format($scfg, $volname);
5278             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5279         } else {
5280             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5281             $format = qemu_img_format($scfg, $volname);
5282         }
5283
5284         my $size = $drive->{size} / 1024;
5285         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5286         my $newdrive = $drive;
5287         $newdrive->{format} = $format;
5288         $newdrive->{file} = $newvolid;
5289         my $drivestr = print_drive($newdrive);
5290         $nbd->{$opt}->{drivestr} = $drivestr;
5291         $nbd->{$opt}->{volid} = $newvolid;
5292     }
5293
5294     return $nbd;
5295 }
5296
5297 # see vm_start_nolock for parameters, additionally:
5298 # migrate_opts:
5299 #   storagemap = parsed storage map for allocating NBD disks
5300 sub vm_start {
5301     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5302
5303     return PVE::QemuConfig->lock_config($vmid, sub {
5304         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5305
5306         die "you can't start a vm if it's a template\n"
5307             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5308
5309         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5310         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5311
5312         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5313
5314         if ($has_backup_lock && $running) {
5315             # a backup is currently running, attempt to start the guest in the
5316             # existing QEMU instance
5317             return vm_resume($vmid);
5318         }
5319
5320         PVE::QemuConfig->check_lock($conf)
5321             if !($params->{skiplock} || $has_suspended_lock);
5322
5323         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5324
5325         die "VM $vmid already running\n" if $running;
5326
5327         if (my $storagemap = $migrate_opts->{storagemap}) {
5328             my $replicated = $migrate_opts->{replicated_volumes};
5329             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5330             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5331
5332             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5333                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5334             }
5335         }
5336
5337         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5338     });
5339 }
5340
5341
5342 # params:
5343 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5344 #   skiplock => 0/1, skip checking for config lock
5345 #   skiptemplate => 0/1, skip checking whether VM is template
5346 #   forcemachine => to force Qemu machine (rollback/migration)
5347 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5348 #   timeout => in seconds
5349 #   paused => start VM in paused state (backup)
5350 #   resume => resume from hibernation
5351 #   pbs-backing => {
5352 #      sata0 => {
5353 #         repository
5354 #         snapshot
5355 #         keyfile
5356 #         archive
5357 #      },
5358 #      virtio2 => ...
5359 #   }
5360 # migrate_opts:
5361 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5362 #   migratedfrom => source node
5363 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5364 #   network => CIDR of migration network
5365 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5366 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5367 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5368 #   tpmstate_vol => new volid of tpmstate0, not yet contained in config
5369 sub vm_start_nolock {
5370     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5371
5372     my $statefile = $params->{statefile};
5373     my $resume = $params->{resume};
5374
5375     my $migratedfrom = $migrate_opts->{migratedfrom};
5376     my $migration_type = $migrate_opts->{type};
5377
5378     my $res = {};
5379
5380     # clean up leftover reboot request files
5381     eval { clear_reboot_request($vmid); };
5382     warn $@ if $@;
5383
5384     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5385         vmconfig_apply_pending($vmid, $conf, $storecfg);
5386         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5387     }
5388
5389     # don't regenerate the ISO if the VM is started as part of a live migration
5390     # this way we can reuse the old ISO with the correct config
5391     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5392
5393     # override TPM state vol if migrated, conf is out of date still
5394     if (my $tpmvol = $migrate_opts->{tpmstate_vol}) {
5395         my $parsed = parse_drive("tpmstate0", $conf->{tpmstate0});
5396         $parsed->{file} = $tpmvol;
5397         $conf->{tpmstate0} = print_drive($parsed);
5398     }
5399
5400     my $defaults = load_defaults();
5401
5402     # set environment variable useful inside network script
5403     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5404
5405     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5406
5407     my $forcemachine = $params->{forcemachine};
5408     my $forcecpu = $params->{forcecpu};
5409     if ($resume) {
5410         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5411         $forcemachine = $conf->{runningmachine};
5412         $forcecpu = $conf->{runningcpu};
5413         print "Resuming suspended VM\n";
5414     }
5415
5416     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5417         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5418
5419     my $migration_ip;
5420     my $get_migration_ip = sub {
5421         my ($nodename) = @_;
5422
5423         return $migration_ip if defined($migration_ip);
5424
5425         my $cidr = $migrate_opts->{network};
5426
5427         if (!defined($cidr)) {
5428             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5429             $cidr = $dc_conf->{migration}->{network};
5430         }
5431
5432         if (defined($cidr)) {
5433             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5434
5435             die "could not get IP: no address configured on local " .
5436                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5437
5438             die "could not get IP: multiple addresses configured on local " .
5439                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5440
5441             $migration_ip = @$ips[0];
5442         }
5443
5444         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5445             if !defined($migration_ip);
5446
5447         return $migration_ip;
5448     };
5449
5450     my $migrate_uri;
5451     if ($statefile) {
5452         if ($statefile eq 'tcp') {
5453             my $localip = "localhost";
5454             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5455             my $nodename = nodename();
5456
5457             if (!defined($migration_type)) {
5458                 if (defined($datacenterconf->{migration}->{type})) {
5459                     $migration_type = $datacenterconf->{migration}->{type};
5460                 } else {
5461                     $migration_type = 'secure';
5462                 }
5463             }
5464
5465             if ($migration_type eq 'insecure') {
5466                 $localip = $get_migration_ip->($nodename);
5467                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5468             }
5469
5470             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5471             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5472             $migrate_uri = "tcp:${localip}:${migrate_port}";
5473             push @$cmd, '-incoming', $migrate_uri;
5474             push @$cmd, '-S';
5475
5476         } elsif ($statefile eq 'unix') {
5477             # should be default for secure migrations as a ssh TCP forward
5478             # tunnel is not deterministic reliable ready and fails regurarly
5479             # to set up in time, so use UNIX socket forwards
5480             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5481             unlink $socket_addr;
5482
5483             $migrate_uri = "unix:$socket_addr";
5484
5485             push @$cmd, '-incoming', $migrate_uri;
5486             push @$cmd, '-S';
5487
5488         } elsif (-e $statefile) {
5489             push @$cmd, '-loadstate', $statefile;
5490         } else {
5491             my $statepath = PVE::Storage::path($storecfg, $statefile);
5492             push @$vollist, $statefile;
5493             push @$cmd, '-loadstate', $statepath;
5494         }
5495     } elsif ($params->{paused}) {
5496         push @$cmd, '-S';
5497     }
5498
5499     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5500
5501     my $pci_devices = {}; # host pci devices
5502     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5503         my $dev = $conf->{"hostpci$i"} or next;
5504         $pci_devices->{$i} = parse_hostpci($dev);
5505     }
5506
5507     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5508     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5509
5510     # map to a flat list of pci ids
5511     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5512
5513     # reserve all PCI IDs before actually doing anything with them
5514     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5515
5516     eval {
5517         for my $id (sort keys %$pci_devices) {
5518             my $d = $pci_devices->{$id};
5519             for my $dev ($d->{pciid}->@*) {
5520                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5521             }
5522         }
5523     };
5524     if (my $err = $@) {
5525         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5526         warn $@ if $@;
5527         die $err;
5528     }
5529
5530     PVE::Storage::activate_volumes($storecfg, $vollist);
5531
5532     eval {
5533         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5534     };
5535     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5536     # timeout should be more than enough here...
5537     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5538
5539     my $cpuunits = get_cpuunits($conf);
5540
5541     my %run_params = (
5542         timeout => $statefile ? undef : $start_timeout,
5543         umask => 0077,
5544         noerr => 1,
5545     );
5546
5547     # when migrating, prefix QEMU output so other side can pick up any
5548     # errors that might occur and show the user
5549     if ($migratedfrom) {
5550         $run_params{quiet} = 1;
5551         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5552     }
5553
5554     my %systemd_properties = (
5555         Slice => 'qemu.slice',
5556         KillMode => 'process',
5557         SendSIGKILL => 0,
5558         TimeoutStopUSec => ULONG_MAX, # infinity
5559     );
5560
5561     if (PVE::CGroup::cgroup_mode() == 2) {
5562         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5563         $systemd_properties{CPUWeight} = $cpuunits;
5564     } else {
5565         $systemd_properties{CPUShares} = $cpuunits;
5566     }
5567
5568     if (my $cpulimit = $conf->{cpulimit}) {
5569         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5570     }
5571     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5572
5573     my $run_qemu = sub {
5574         PVE::Tools::run_fork sub {
5575             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5576
5577             my $tpmpid;
5578             if (my $tpm = $conf->{tpmstate0}) {
5579                 # start the TPM emulator so QEMU can connect on start
5580                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5581             }
5582
5583             my $exitcode = run_command($cmd, %run_params);
5584             if ($exitcode) {
5585                 if ($tpmpid) {
5586                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5587                     kill 'TERM', $tpmpid;
5588                 }
5589                 die "QEMU exited with code $exitcode\n";
5590             }
5591         };
5592     };
5593
5594     if ($conf->{hugepages}) {
5595
5596         my $code = sub {
5597             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5598             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5599
5600             PVE::QemuServer::Memory::hugepages_mount();
5601             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5602
5603             eval { $run_qemu->() };
5604             if (my $err = $@) {
5605                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5606                     if !$conf->{keephugepages};
5607                 die $err;
5608             }
5609
5610             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5611                 if !$conf->{keephugepages};
5612         };
5613         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5614
5615     } else {
5616         eval { $run_qemu->() };
5617     }
5618
5619     if (my $err = $@) {
5620         # deactivate volumes if start fails
5621         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5622         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5623
5624         die "start failed: $err";
5625     }
5626
5627     # re-reserve all PCI IDs now that we can know the actual VM PID
5628     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5629     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5630     warn $@ if $@;
5631
5632     print "migration listens on $migrate_uri\n" if $migrate_uri;
5633     $res->{migrate_uri} = $migrate_uri;
5634
5635     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5636         eval { mon_cmd($vmid, "cont"); };
5637         warn $@ if $@;
5638     }
5639
5640     #start nbd server for storage migration
5641     if (my $nbd = $migrate_opts->{nbd}) {
5642         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5643
5644         my $migrate_storage_uri;
5645         # nbd_protocol_version > 0 for unix socket support
5646         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5647             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5648             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5649             $migrate_storage_uri = "nbd:unix:$socket_path";
5650         } else {
5651             my $nodename = nodename();
5652             my $localip = $get_migration_ip->($nodename);
5653             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5654             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5655
5656             mon_cmd($vmid, "nbd-server-start", addr => {
5657                 type => 'inet',
5658                 data => {
5659                     host => "${localip}",
5660                     port => "${storage_migrate_port}",
5661                 },
5662             });
5663             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5664             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5665         }
5666
5667         $res->{migrate_storage_uri} = $migrate_storage_uri;
5668
5669         foreach my $opt (sort keys %$nbd) {
5670             my $drivestr = $nbd->{$opt}->{drivestr};
5671             my $volid = $nbd->{$opt}->{volid};
5672             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5673             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5674             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5675             print "re-using replicated volume: $opt - $volid\n"
5676                 if $nbd->{$opt}->{replicated};
5677
5678             $res->{drives}->{$opt} = $nbd->{$opt};
5679             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5680         }
5681     }
5682
5683     if ($migratedfrom) {
5684         eval {
5685             set_migration_caps($vmid);
5686         };
5687         warn $@ if $@;
5688
5689         if ($spice_port) {
5690             print "spice listens on port $spice_port\n";
5691             $res->{spice_port} = $spice_port;
5692             if ($migrate_opts->{spice_ticket}) {
5693                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5694                     $migrate_opts->{spice_ticket});
5695                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5696             }
5697         }
5698
5699     } else {
5700         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5701             if !$statefile && $conf->{balloon};
5702
5703         foreach my $opt (keys %$conf) {
5704             next if $opt !~  m/^net\d+$/;
5705             my $nicconf = parse_net($conf->{$opt});
5706             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5707         }
5708     }
5709
5710     mon_cmd($vmid, 'qom-set',
5711                 path => "machine/peripheral/balloon0",
5712                 property => "guest-stats-polling-interval",
5713                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5714
5715     if ($resume) {
5716         print "Resumed VM, removing state\n";
5717         if (my $vmstate = $conf->{vmstate}) {
5718             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5719             PVE::Storage::vdisk_free($storecfg, $vmstate);
5720         }
5721         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5722         PVE::QemuConfig->write_config($vmid, $conf);
5723     }
5724
5725     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5726
5727     return $res;
5728 }
5729
5730 sub vm_commandline {
5731     my ($storecfg, $vmid, $snapname) = @_;
5732
5733     my $conf = PVE::QemuConfig->load_config($vmid);
5734
5735     my ($forcemachine, $forcecpu);
5736     if ($snapname) {
5737         my $snapshot = $conf->{snapshots}->{$snapname};
5738         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5739
5740         # check for machine or CPU overrides in snapshot
5741         $forcemachine = $snapshot->{runningmachine};
5742         $forcecpu = $snapshot->{runningcpu};
5743
5744         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5745
5746         $conf = $snapshot;
5747     }
5748
5749     my $defaults = load_defaults();
5750
5751     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5752
5753     return PVE::Tools::cmd2string($cmd);
5754 }
5755
5756 sub vm_reset {
5757     my ($vmid, $skiplock) = @_;
5758
5759     PVE::QemuConfig->lock_config($vmid, sub {
5760
5761         my $conf = PVE::QemuConfig->load_config($vmid);
5762
5763         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5764
5765         mon_cmd($vmid, "system_reset");
5766     });
5767 }
5768
5769 sub get_vm_volumes {
5770     my ($conf) = @_;
5771
5772     my $vollist = [];
5773     foreach_volid($conf, sub {
5774         my ($volid, $attr) = @_;
5775
5776         return if $volid =~ m|^/|;
5777
5778         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5779         return if !$sid;
5780
5781         push @$vollist, $volid;
5782     });
5783
5784     return $vollist;
5785 }
5786
5787 sub vm_stop_cleanup {
5788     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5789
5790     eval {
5791
5792         if (!$keepActive) {
5793             my $vollist = get_vm_volumes($conf);
5794             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5795
5796             if (my $tpmdrive = $conf->{tpmstate0}) {
5797                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5798                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5799                 if ($storeid) {
5800                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5801                 }
5802             }
5803         }
5804
5805         foreach my $ext (qw(mon qmp pid vnc qga)) {
5806             unlink "/var/run/qemu-server/${vmid}.$ext";
5807         }
5808
5809         if ($conf->{ivshmem}) {
5810             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5811             # just delete it for now, VMs which have this already open do not
5812             # are affected, but new VMs will get a separated one. If this
5813             # becomes an issue we either add some sort of ref-counting or just
5814             # add a "don't delete on stop" flag to the ivshmem format.
5815             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5816         }
5817
5818         my $ids = [];
5819         foreach my $key (keys %$conf) {
5820             next if $key !~ m/^hostpci(\d+)$/;
5821             my $hostpciindex = $1;
5822             my $d = parse_hostpci($conf->{$key});
5823             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5824
5825             foreach my $pci (@{$d->{pciid}}) {
5826                 my $pciid = $pci->{id};
5827                 push @$ids, $pci->{id};
5828                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5829             }
5830         }
5831         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5832
5833         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5834     };
5835     warn $@ if $@; # avoid errors - just warn
5836 }
5837
5838 # call only in locked context
5839 sub _do_vm_stop {
5840     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5841
5842     my $pid = check_running($vmid, $nocheck);
5843     return if !$pid;
5844
5845     my $conf;
5846     if (!$nocheck) {
5847         $conf = PVE::QemuConfig->load_config($vmid);
5848         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5849         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5850             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5851             $timeout = $opts->{down} if $opts->{down};
5852         }
5853         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5854     }
5855
5856     eval {
5857         if ($shutdown) {
5858             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5859                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5860             } else {
5861                 mon_cmd($vmid, "system_powerdown");
5862             }
5863         } else {
5864             mon_cmd($vmid, "quit");
5865         }
5866     };
5867     my $err = $@;
5868
5869     if (!$err) {
5870         $timeout = 60 if !defined($timeout);
5871
5872         my $count = 0;
5873         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5874             $count++;
5875             sleep 1;
5876         }
5877
5878         if ($count >= $timeout) {
5879             if ($force) {
5880                 warn "VM still running - terminating now with SIGTERM\n";
5881                 kill 15, $pid;
5882             } else {
5883                 die "VM quit/powerdown failed - got timeout\n";
5884             }
5885         } else {
5886             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5887             return;
5888         }
5889     } else {
5890        if (!check_running($vmid, $nocheck)) {
5891             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5892             return;
5893        }
5894        if ($force) {
5895             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5896             kill 15, $pid;
5897         } else {
5898             die "VM quit/powerdown failed\n";
5899         }
5900     }
5901
5902     # wait again
5903     $timeout = 10;
5904
5905     my $count = 0;
5906     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5907         $count++;
5908         sleep 1;
5909     }
5910
5911     if ($count >= $timeout) {
5912         warn "VM still running - terminating now with SIGKILL\n";
5913         kill 9, $pid;
5914         sleep 1;
5915     }
5916
5917     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5918 }
5919
5920 # Note: use $nocheck to skip tests if VM configuration file exists.
5921 # We need that when migration VMs to other nodes (files already moved)
5922 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5923 sub vm_stop {
5924     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5925
5926     $force = 1 if !defined($force) && !$shutdown;
5927
5928     if ($migratedfrom){
5929         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5930         kill 15, $pid if $pid;
5931         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5932         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5933         return;
5934     }
5935
5936     PVE::QemuConfig->lock_config($vmid, sub {
5937         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5938    });
5939 }
5940
5941 sub vm_reboot {
5942     my ($vmid, $timeout) = @_;
5943
5944     PVE::QemuConfig->lock_config($vmid, sub {
5945         eval {
5946
5947             # only reboot if running, as qmeventd starts it again on a stop event
5948             return if !check_running($vmid);
5949
5950             create_reboot_request($vmid);
5951
5952             my $storecfg = PVE::Storage::config();
5953             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5954
5955         };
5956         if (my $err = $@) {
5957             # avoid that the next normal shutdown will be confused for a reboot
5958             clear_reboot_request($vmid);
5959             die $err;
5960         }
5961    });
5962 }
5963
5964 # note: if using the statestorage parameter, the caller has to check privileges
5965 sub vm_suspend {
5966     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5967
5968     my $conf;
5969     my $path;
5970     my $storecfg;
5971     my $vmstate;
5972
5973     PVE::QemuConfig->lock_config($vmid, sub {
5974
5975         $conf = PVE::QemuConfig->load_config($vmid);
5976
5977         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5978         PVE::QemuConfig->check_lock($conf)
5979             if !($skiplock || $is_backing_up);
5980
5981         die "cannot suspend to disk during backup\n"
5982             if $is_backing_up && $includestate;
5983
5984         if ($includestate) {
5985             $conf->{lock} = 'suspending';
5986             my $date = strftime("%Y-%m-%d", localtime(time()));
5987             $storecfg = PVE::Storage::config();
5988             if (!$statestorage) {
5989                 $statestorage = find_vmstate_storage($conf, $storecfg);
5990                 # check permissions for the storage
5991                 my $rpcenv = PVE::RPCEnvironment::get();
5992                 if ($rpcenv->{type} ne 'cli') {
5993                     my $authuser = $rpcenv->get_user();
5994                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5995                 }
5996             }
5997
5998
5999             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6000                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6001             $path = PVE::Storage::path($storecfg, $vmstate);
6002             PVE::QemuConfig->write_config($vmid, $conf);
6003         } else {
6004             mon_cmd($vmid, "stop");
6005         }
6006     });
6007
6008     if ($includestate) {
6009         # save vm state
6010         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6011
6012         eval {
6013             set_migration_caps($vmid, 1);
6014             mon_cmd($vmid, "savevm-start", statefile => $path);
6015             for(;;) {
6016                 my $state = mon_cmd($vmid, "query-savevm");
6017                 if (!$state->{status}) {
6018                     die "savevm not active\n";
6019                 } elsif ($state->{status} eq 'active') {
6020                     sleep(1);
6021                     next;
6022                 } elsif ($state->{status} eq 'completed') {
6023                     print "State saved, quitting\n";
6024                     last;
6025                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6026                     die "query-savevm failed with error '$state->{error}'\n"
6027                 } else {
6028                     die "query-savevm returned status '$state->{status}'\n";
6029                 }
6030             }
6031         };
6032         my $err = $@;
6033
6034         PVE::QemuConfig->lock_config($vmid, sub {
6035             $conf = PVE::QemuConfig->load_config($vmid);
6036             if ($err) {
6037                 # cleanup, but leave suspending lock, to indicate something went wrong
6038                 eval {
6039                     mon_cmd($vmid, "savevm-end");
6040                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6041                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6042                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6043                     PVE::QemuConfig->write_config($vmid, $conf);
6044                 };
6045                 warn $@ if $@;
6046                 die $err;
6047             }
6048
6049             die "lock changed unexpectedly\n"
6050                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6051
6052             mon_cmd($vmid, "quit");
6053             $conf->{lock} = 'suspended';
6054             PVE::QemuConfig->write_config($vmid, $conf);
6055         });
6056     }
6057 }
6058
6059 sub vm_resume {
6060     my ($vmid, $skiplock, $nocheck) = @_;
6061
6062     PVE::QemuConfig->lock_config($vmid, sub {
6063         my $res = mon_cmd($vmid, 'query-status');
6064         my $resume_cmd = 'cont';
6065         my $reset = 0;
6066
6067         if ($res->{status}) {
6068             return if $res->{status} eq 'running'; # job done, go home
6069             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6070             $reset = 1 if $res->{status} eq 'shutdown';
6071         }
6072
6073         if (!$nocheck) {
6074
6075             my $conf = PVE::QemuConfig->load_config($vmid);
6076
6077             PVE::QemuConfig->check_lock($conf)
6078                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6079         }
6080
6081         if ($reset) {
6082             # required if a VM shuts down during a backup and we get a resume
6083             # request before the backup finishes for example
6084             mon_cmd($vmid, "system_reset");
6085         }
6086         mon_cmd($vmid, $resume_cmd);
6087     });
6088 }
6089
6090 sub vm_sendkey {
6091     my ($vmid, $skiplock, $key) = @_;
6092
6093     PVE::QemuConfig->lock_config($vmid, sub {
6094
6095         my $conf = PVE::QemuConfig->load_config($vmid);
6096
6097         # there is no qmp command, so we use the human monitor command
6098         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6099         die $res if $res ne '';
6100     });
6101 }
6102
6103 # vzdump restore implementaion
6104
6105 sub tar_archive_read_firstfile {
6106     my $archive = shift;
6107
6108     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6109
6110     # try to detect archive type first
6111     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6112         die "unable to open file '$archive'\n";
6113     my $firstfile = <$fh>;
6114     kill 15, $pid;
6115     close $fh;
6116
6117     die "ERROR: archive contaions no data\n" if !$firstfile;
6118     chomp $firstfile;
6119
6120     return $firstfile;
6121 }
6122
6123 sub tar_restore_cleanup {
6124     my ($storecfg, $statfile) = @_;
6125
6126     print STDERR "starting cleanup\n";
6127
6128     if (my $fd = IO::File->new($statfile, "r")) {
6129         while (defined(my $line = <$fd>)) {
6130             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6131                 my $volid = $2;
6132                 eval {
6133                     if ($volid =~ m|^/|) {
6134                         unlink $volid || die 'unlink failed\n';
6135                     } else {
6136                         PVE::Storage::vdisk_free($storecfg, $volid);
6137                     }
6138                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6139                 };
6140                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6141             } else {
6142                 print STDERR "unable to parse line in statfile - $line";
6143             }
6144         }
6145         $fd->close();
6146     }
6147 }
6148
6149 sub restore_file_archive {
6150     my ($archive, $vmid, $user, $opts) = @_;
6151
6152     return restore_vma_archive($archive, $vmid, $user, $opts)
6153         if $archive eq '-';
6154
6155     my $info = PVE::Storage::archive_info($archive);
6156     my $format = $opts->{format} // $info->{format};
6157     my $comp = $info->{compression};
6158
6159     # try to detect archive format
6160     if ($format eq 'tar') {
6161         return restore_tar_archive($archive, $vmid, $user, $opts);
6162     } else {
6163         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6164     }
6165 }
6166
6167 # hepler to remove disks that will not be used after restore
6168 my $restore_cleanup_oldconf = sub {
6169     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6170
6171     PVE::QemuConfig->foreach_volume($oldconf, sub {
6172         my ($ds, $drive) = @_;
6173
6174         return if drive_is_cdrom($drive, 1);
6175
6176         my $volid = $drive->{file};
6177         return if !$volid || $volid =~ m|^/|;
6178
6179         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6180         return if !$path || !$owner || ($owner != $vmid);
6181
6182         # Note: only delete disk we want to restore
6183         # other volumes will become unused
6184         if ($virtdev_hash->{$ds}) {
6185             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6186             if (my $err = $@) {
6187                 warn $err;
6188             }
6189         }
6190     });
6191
6192     # delete vmstate files, after the restore we have no snapshots anymore
6193     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6194         my $snap = $oldconf->{snapshots}->{$snapname};
6195         if ($snap->{vmstate}) {
6196             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6197             if (my $err = $@) {
6198                 warn $err;
6199             }
6200         }
6201     }
6202 };
6203
6204 # Helper to parse vzdump backup device hints
6205 #
6206 # $rpcenv: Environment, used to ckeck storage permissions
6207 # $user: User ID, to check storage permissions
6208 # $storecfg: Storage configuration
6209 # $fh: the file handle for reading the configuration
6210 # $devinfo: should contain device sizes for all backu-up'ed devices
6211 # $options: backup options (pool, default storage)
6212 #
6213 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6214 my $parse_backup_hints = sub {
6215     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6216
6217     my $virtdev_hash = {};
6218
6219     while (defined(my $line = <$fh>)) {
6220         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6221             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6222             die "archive does not contain data for drive '$virtdev'\n"
6223                 if !$devinfo->{$devname};
6224
6225             if (defined($options->{storage})) {
6226                 $storeid = $options->{storage} || 'local';
6227             } elsif (!$storeid) {
6228                 $storeid = 'local';
6229             }
6230             $format = 'raw' if !$format;
6231             $devinfo->{$devname}->{devname} = $devname;
6232             $devinfo->{$devname}->{virtdev} = $virtdev;
6233             $devinfo->{$devname}->{format} = $format;
6234             $devinfo->{$devname}->{storeid} = $storeid;
6235
6236             # check permission on storage
6237             my $pool = $options->{pool}; # todo: do we need that?
6238             if ($user ne 'root@pam') {
6239                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6240             }
6241
6242             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6243         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6244             my $virtdev = $1;
6245             my $drive = parse_drive($virtdev, $2);
6246             if (drive_is_cloudinit($drive)) {
6247                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6248                 $storeid = $options->{storage} if defined ($options->{storage});
6249                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6250                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6251
6252                 $virtdev_hash->{$virtdev} = {
6253                     format => $format,
6254                     storeid => $storeid,
6255                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6256                     is_cloudinit => 1,
6257                 };
6258             }
6259         }
6260     }
6261
6262     return $virtdev_hash;
6263 };
6264
6265 # Helper to allocate and activate all volumes required for a restore
6266 #
6267 # $storecfg: Storage configuration
6268 # $virtdev_hash: as returned by parse_backup_hints()
6269 #
6270 # Returns: { $virtdev => $volid }
6271 my $restore_allocate_devices = sub {
6272     my ($storecfg, $virtdev_hash, $vmid) = @_;
6273
6274     my $map = {};
6275     foreach my $virtdev (sort keys %$virtdev_hash) {
6276         my $d = $virtdev_hash->{$virtdev};
6277         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6278         my $storeid = $d->{storeid};
6279         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6280
6281         # test if requested format is supported
6282         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6283         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6284         $d->{format} = $defFormat if !$supported;
6285
6286         my $name;
6287         if ($d->{is_cloudinit}) {
6288             $name = "vm-$vmid-cloudinit";
6289             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6290             if ($scfg->{path}) {
6291                 $name .= ".$d->{format}";
6292             }
6293         }
6294
6295         my $volid = PVE::Storage::vdisk_alloc(
6296             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6297
6298         print STDERR "new volume ID is '$volid'\n";
6299         $d->{volid} = $volid;
6300
6301         PVE::Storage::activate_volumes($storecfg, [$volid]);
6302
6303         $map->{$virtdev} = $volid;
6304     }
6305
6306     return $map;
6307 };
6308
6309 sub restore_update_config_line {
6310     my ($cookie, $map, $line, $unique) = @_;
6311
6312     return '' if $line =~ m/^\#qmdump\#/;
6313     return '' if $line =~ m/^\#vzdump\#/;
6314     return '' if $line =~ m/^lock:/;
6315     return '' if $line =~ m/^unused\d+:/;
6316     return '' if $line =~ m/^parent:/;
6317
6318     my $res = '';
6319
6320     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6321     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6322         # try to convert old 1.X settings
6323         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6324         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6325             my ($model, $macaddr) = split(/\=/, $devconfig);
6326             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6327             my $net = {
6328                 model => $model,
6329                 bridge => "vmbr$ind",
6330                 macaddr => $macaddr,
6331             };
6332             my $netstr = print_net($net);
6333
6334             $res .= "net$cookie->{netcount}: $netstr\n";
6335             $cookie->{netcount}++;
6336         }
6337     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6338         my ($id, $netstr) = ($1, $2);
6339         my $net = parse_net($netstr);
6340         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6341         $netstr = print_net($net);
6342         $res .= "$id: $netstr\n";
6343     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6344         my $virtdev = $1;
6345         my $value = $3;
6346         my $di = parse_drive($virtdev, $value);
6347         if (defined($di->{backup}) && !$di->{backup}) {
6348             $res .= "#$line";
6349         } elsif ($map->{$virtdev}) {
6350             delete $di->{format}; # format can change on restore
6351             $di->{file} = $map->{$virtdev};
6352             $value = print_drive($di);
6353             $res .= "$virtdev: $value\n";
6354         } else {
6355             $res .= $line;
6356         }
6357     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6358         my $vmgenid = $1;
6359         if ($vmgenid ne '0') {
6360             # always generate a new vmgenid if there was a valid one setup
6361             $vmgenid = generate_uuid();
6362         }
6363         $res .= "vmgenid: $vmgenid\n";
6364     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6365         my ($uuid, $uuid_str);
6366         UUID::generate($uuid);
6367         UUID::unparse($uuid, $uuid_str);
6368         my $smbios1 = parse_smbios1($2);
6369         $smbios1->{uuid} = $uuid_str;
6370         $res .= $1.print_smbios1($smbios1)."\n";
6371     } else {
6372         $res .= $line;
6373     }
6374
6375     return $res;
6376 }
6377
6378 my $restore_deactivate_volumes = sub {
6379     my ($storecfg, $devinfo) = @_;
6380
6381     my $vollist = [];
6382     foreach my $devname (keys %$devinfo) {
6383         my $volid = $devinfo->{$devname}->{volid};
6384         push @$vollist, $volid if $volid;
6385     }
6386
6387     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6388 };
6389
6390 my $restore_destroy_volumes = sub {
6391     my ($storecfg, $devinfo) = @_;
6392
6393     foreach my $devname (keys %$devinfo) {
6394         my $volid = $devinfo->{$devname}->{volid};
6395         next if !$volid;
6396         eval {
6397             if ($volid =~ m|^/|) {
6398                 unlink $volid || die 'unlink failed\n';
6399             } else {
6400                 PVE::Storage::vdisk_free($storecfg, $volid);
6401             }
6402             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6403         };
6404         print STDERR "unable to cleanup '$volid' - $@" if $@;
6405     }
6406 };
6407
6408 sub scan_volids {
6409     my ($cfg, $vmid) = @_;
6410
6411     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6412
6413     my $volid_hash = {};
6414     foreach my $storeid (keys %$info) {
6415         foreach my $item (@{$info->{$storeid}}) {
6416             next if !($item->{volid} && $item->{size});
6417             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6418             $volid_hash->{$item->{volid}} = $item;
6419         }
6420     }
6421
6422     return $volid_hash;
6423 }
6424
6425 sub update_disk_config {
6426     my ($vmid, $conf, $volid_hash) = @_;
6427
6428     my $changes;
6429     my $prefix = "VM $vmid";
6430
6431     # used and unused disks
6432     my $referenced = {};
6433
6434     # Note: it is allowed to define multiple storages with same path (alias), so
6435     # we need to check both 'volid' and real 'path' (two different volid can point
6436     # to the same path).
6437
6438     my $referencedpath = {};
6439
6440     # update size info
6441     PVE::QemuConfig->foreach_volume($conf, sub {
6442         my ($opt, $drive) = @_;
6443
6444         my $volid = $drive->{file};
6445         return if !$volid;
6446         my $volume = $volid_hash->{$volid};
6447
6448         # mark volid as "in-use" for next step
6449         $referenced->{$volid} = 1;
6450         if ($volume && (my $path = $volume->{path})) {
6451             $referencedpath->{$path} = 1;
6452         }
6453
6454         return if drive_is_cdrom($drive);
6455         return if !$volume;
6456
6457         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6458         if (defined($updated)) {
6459             $changes = 1;
6460             $conf->{$opt} = print_drive($updated);
6461             print "$prefix ($opt): $msg\n";
6462         }
6463     });
6464
6465     # remove 'unusedX' entry if volume is used
6466     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6467         my ($opt, $drive) = @_;
6468
6469         my $volid = $drive->{file};
6470         return if !$volid;
6471
6472         my $path;
6473         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6474         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6475             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6476             $changes = 1;
6477             delete $conf->{$opt};
6478         }
6479
6480         $referenced->{$volid} = 1;
6481         $referencedpath->{$path} = 1 if $path;
6482     });
6483
6484     foreach my $volid (sort keys %$volid_hash) {
6485         next if $volid =~ m/vm-$vmid-state-/;
6486         next if $referenced->{$volid};
6487         my $path = $volid_hash->{$volid}->{path};
6488         next if !$path; # just to be sure
6489         next if $referencedpath->{$path};
6490         $changes = 1;
6491         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6492         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6493         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6494     }
6495
6496     return $changes;
6497 }
6498
6499 sub rescan {
6500     my ($vmid, $nolock, $dryrun) = @_;
6501
6502     my $cfg = PVE::Storage::config();
6503
6504     print "rescan volumes...\n";
6505     my $volid_hash = scan_volids($cfg, $vmid);
6506
6507     my $updatefn =  sub {
6508         my ($vmid) = @_;
6509
6510         my $conf = PVE::QemuConfig->load_config($vmid);
6511
6512         PVE::QemuConfig->check_lock($conf);
6513
6514         my $vm_volids = {};
6515         foreach my $volid (keys %$volid_hash) {
6516             my $info = $volid_hash->{$volid};
6517             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6518         }
6519
6520         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6521
6522         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6523     };
6524
6525     if (defined($vmid)) {
6526         if ($nolock) {
6527             &$updatefn($vmid);
6528         } else {
6529             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6530         }
6531     } else {
6532         my $vmlist = config_list();
6533         foreach my $vmid (keys %$vmlist) {
6534             if ($nolock) {
6535                 &$updatefn($vmid);
6536             } else {
6537                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6538             }
6539         }
6540     }
6541 }
6542
6543 sub restore_proxmox_backup_archive {
6544     my ($archive, $vmid, $user, $options) = @_;
6545
6546     my $storecfg = PVE::Storage::config();
6547
6548     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6549     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6550
6551     my $fingerprint = $scfg->{fingerprint};
6552     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6553
6554     my $repo = PVE::PBSClient::get_repository($scfg);
6555
6556     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6557     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6558     local $ENV{PBS_PASSWORD} = $password;
6559     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6560
6561     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6562         PVE::Storage::parse_volname($storecfg, $archive);
6563
6564     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6565
6566     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6567
6568     my $tmpdir = "/var/tmp/vzdumptmp$$";
6569     rmtree $tmpdir;
6570     mkpath $tmpdir;
6571
6572     my $conffile = PVE::QemuConfig->config_file($vmid);
6573      # disable interrupts (always do cleanups)
6574     local $SIG{INT} =
6575         local $SIG{TERM} =
6576         local $SIG{QUIT} =
6577         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6578
6579     # Note: $oldconf is undef if VM does not exists
6580     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6581     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6582     my $new_conf_raw = '';
6583
6584     my $rpcenv = PVE::RPCEnvironment::get();
6585     my $devinfo = {};
6586
6587     eval {
6588         # enable interrupts
6589         local $SIG{INT} =
6590             local $SIG{TERM} =
6591             local $SIG{QUIT} =
6592             local $SIG{HUP} =
6593             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6594
6595         my $cfgfn = "$tmpdir/qemu-server.conf";
6596         my $firewall_config_fn = "$tmpdir/fw.conf";
6597         my $index_fn = "$tmpdir/index.json";
6598
6599         my $cmd = "restore";
6600
6601         my $param = [$pbs_backup_name, "index.json", $index_fn];
6602         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6603         my $index = PVE::Tools::file_get_contents($index_fn);
6604         $index = decode_json($index);
6605
6606         # print Dumper($index);
6607         foreach my $info (@{$index->{files}}) {
6608             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6609                 my $devname = $1;
6610                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6611                     $devinfo->{$devname}->{size} = $1;
6612                 } else {
6613                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6614                 }
6615             }
6616         }
6617
6618         my $is_qemu_server_backup = scalar(
6619             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6620         );
6621         if (!$is_qemu_server_backup) {
6622             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6623         }
6624         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6625
6626         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6627         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6628
6629         if ($has_firewall_config) {
6630             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6631             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6632
6633             my $pve_firewall_dir = '/etc/pve/firewall';
6634             mkdir $pve_firewall_dir; # make sure the dir exists
6635             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6636         }
6637
6638         my $fh = IO::File->new($cfgfn, "r") ||
6639             die "unable to read qemu-server.conf - $!\n";
6640
6641         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6642
6643         # fixme: rate limit?
6644
6645         # create empty/temp config
6646         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6647
6648         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6649
6650         # allocate volumes
6651         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6652
6653         foreach my $virtdev (sort keys %$virtdev_hash) {
6654             my $d = $virtdev_hash->{$virtdev};
6655             next if $d->{is_cloudinit}; # no need to restore cloudinit
6656
6657             # this fails if storage is unavailable
6658             my $volid = $d->{volid};
6659             my $path = PVE::Storage::path($storecfg, $volid);
6660
6661             # for live-restore we only want to preload the efidisk and TPM state
6662             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6663
6664             my $pbs_restore_cmd = [
6665                 '/usr/bin/pbs-restore',
6666                 '--repository', $repo,
6667                 $pbs_backup_name,
6668                 "$d->{devname}.img.fidx",
6669                 $path,
6670                 '--verbose',
6671                 ];
6672
6673             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6674             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6675
6676             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6677                 push @$pbs_restore_cmd, '--skip-zero';
6678             }
6679
6680             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6681             print "restore proxmox backup image: $dbg_cmdstring\n";
6682             run_command($pbs_restore_cmd);
6683         }
6684
6685         $fh->seek(0, 0) || die "seek failed - $!\n";
6686
6687         my $cookie = { netcount => 0 };
6688         while (defined(my $line = <$fh>)) {
6689             $new_conf_raw .= restore_update_config_line(
6690                 $cookie,
6691                 $map,
6692                 $line,
6693                 $options->{unique},
6694             );
6695         }
6696
6697         $fh->close();
6698     };
6699     my $err = $@;
6700
6701     if ($err || !$options->{live}) {
6702         $restore_deactivate_volumes->($storecfg, $devinfo);
6703     }
6704
6705     rmtree $tmpdir;
6706
6707     if ($err) {
6708         $restore_destroy_volumes->($storecfg, $devinfo);
6709         die $err;
6710     }
6711
6712     if ($options->{live}) {
6713         # keep lock during live-restore
6714         $new_conf_raw .= "\nlock: create";
6715     }
6716
6717     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6718
6719     PVE::Cluster::cfs_update(); # make sure we read new file
6720
6721     eval { rescan($vmid, 1); };
6722     warn $@ if $@;
6723
6724     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6725
6726     if ($options->{live}) {
6727         # enable interrupts
6728         local $SIG{INT} =
6729             local $SIG{TERM} =
6730             local $SIG{QUIT} =
6731             local $SIG{HUP} =
6732             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6733
6734         my $conf = PVE::QemuConfig->load_config($vmid);
6735         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6736
6737         # these special drives are already restored before start
6738         delete $devinfo->{'drive-efidisk0'};
6739         delete $devinfo->{'drive-tpmstate0-backup'};
6740         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6741
6742         PVE::QemuConfig->remove_lock($vmid, "create");
6743     }
6744 }
6745
6746 sub pbs_live_restore {
6747     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6748
6749     print "starting VM for live-restore\n";
6750     print "repository: '$repo', snapshot: '$snap'\n";
6751
6752     my $pbs_backing = {};
6753     for my $ds (keys %$restored_disks) {
6754         $ds =~ m/^drive-(.*)$/;
6755         my $confname = $1;
6756         $pbs_backing->{$confname} = {
6757             repository => $repo,
6758             snapshot => $snap,
6759             archive => "$ds.img.fidx",
6760         };
6761         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6762
6763         my $drive = parse_drive($confname, $conf->{$confname});
6764         print "restoring '$ds' to '$drive->{file}'\n";
6765     }
6766
6767     my $drives_streamed = 0;
6768     eval {
6769         # make sure HA doesn't interrupt our restore by stopping the VM
6770         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6771             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6772         }
6773
6774         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6775         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6776         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6777
6778         my $qmeventd_fd = register_qmeventd_handle($vmid);
6779
6780         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6781         # this will effectively collapse the backing image chain consisting of
6782         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6783         # removes itself once all backing images vanish with 'auto-remove=on')
6784         my $jobs = {};
6785         for my $ds (sort keys %$restored_disks) {
6786             my $job_id = "restore-$ds";
6787             mon_cmd($vmid, 'block-stream',
6788                 'job-id' => $job_id,
6789                 device => "$ds",
6790             );
6791             $jobs->{$job_id} = {};
6792         }
6793
6794         mon_cmd($vmid, 'cont');
6795         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6796
6797         print "restore-drive jobs finished successfully, removing all tracking block devices"
6798             ." to disconnect from Proxmox Backup Server\n";
6799
6800         for my $ds (sort keys %$restored_disks) {
6801             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6802         }
6803
6804         close($qmeventd_fd);
6805     };
6806
6807     my $err = $@;
6808
6809     if ($err) {
6810         warn "An error occured during live-restore: $err\n";
6811         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6812         die "live-restore failed\n";
6813     }
6814 }
6815
6816 sub restore_vma_archive {
6817     my ($archive, $vmid, $user, $opts, $comp) = @_;
6818
6819     my $readfrom = $archive;
6820
6821     my $cfg = PVE::Storage::config();
6822     my $commands = [];
6823     my $bwlimit = $opts->{bwlimit};
6824
6825     my $dbg_cmdstring = '';
6826     my $add_pipe = sub {
6827         my ($cmd) = @_;
6828         push @$commands, $cmd;
6829         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6830         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6831         $readfrom = '-';
6832     };
6833
6834     my $input = undef;
6835     if ($archive eq '-') {
6836         $input = '<&STDIN';
6837     } else {
6838         # If we use a backup from a PVE defined storage we also consider that
6839         # storage's rate limit:
6840         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6841         if (defined($volid)) {
6842             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6843             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6844             if ($readlimit) {
6845                 print STDERR "applying read rate limit: $readlimit\n";
6846                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6847                 $add_pipe->($cstream);
6848             }
6849         }
6850     }
6851
6852     if ($comp) {
6853         my $info = PVE::Storage::decompressor_info('vma', $comp);
6854         my $cmd = $info->{decompressor};
6855         push @$cmd, $readfrom;
6856         $add_pipe->($cmd);
6857     }
6858
6859     my $tmpdir = "/var/tmp/vzdumptmp$$";
6860     rmtree $tmpdir;
6861
6862     # disable interrupts (always do cleanups)
6863     local $SIG{INT} =
6864         local $SIG{TERM} =
6865         local $SIG{QUIT} =
6866         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6867
6868     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6869     POSIX::mkfifo($mapfifo, 0600);
6870     my $fifofh;
6871     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6872
6873     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6874
6875     my $oldtimeout;
6876     my $timeout = 5;
6877
6878     my $devinfo = {};
6879
6880     my $rpcenv = PVE::RPCEnvironment::get();
6881
6882     my $conffile = PVE::QemuConfig->config_file($vmid);
6883
6884     # Note: $oldconf is undef if VM does not exist
6885     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6886     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6887     my $new_conf_raw = '';
6888
6889     my %storage_limits;
6890
6891     my $print_devmap = sub {
6892         my $cfgfn = "$tmpdir/qemu-server.conf";
6893
6894         # we can read the config - that is already extracted
6895         my $fh = IO::File->new($cfgfn, "r") ||
6896             die "unable to read qemu-server.conf - $!\n";
6897
6898         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6899         if (-f $fwcfgfn) {
6900             my $pve_firewall_dir = '/etc/pve/firewall';
6901             mkdir $pve_firewall_dir; # make sure the dir exists
6902             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6903         }
6904
6905         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6906
6907         foreach my $info (values %{$virtdev_hash}) {
6908             my $storeid = $info->{storeid};
6909             next if defined($storage_limits{$storeid});
6910
6911             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6912             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6913             $storage_limits{$storeid} = $limit * 1024;
6914         }
6915
6916         foreach my $devname (keys %$devinfo) {
6917             die "found no device mapping information for device '$devname'\n"
6918                 if !$devinfo->{$devname}->{virtdev};
6919         }
6920
6921         # create empty/temp config
6922         if ($oldconf) {
6923             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6924             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6925         }
6926
6927         # allocate volumes
6928         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6929
6930         # print restore information to $fifofh
6931         foreach my $virtdev (sort keys %$virtdev_hash) {
6932             my $d = $virtdev_hash->{$virtdev};
6933             next if $d->{is_cloudinit}; # no need to restore cloudinit
6934
6935             my $storeid = $d->{storeid};
6936             my $volid = $d->{volid};
6937
6938             my $map_opts = '';
6939             if (my $limit = $storage_limits{$storeid}) {
6940                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6941             }
6942
6943             my $write_zeros = 1;
6944             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6945                 $write_zeros = 0;
6946             }
6947
6948             my $path = PVE::Storage::path($cfg, $volid);
6949
6950             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6951
6952             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6953         }
6954
6955         $fh->seek(0, 0) || die "seek failed - $!\n";
6956
6957         my $cookie = { netcount => 0 };
6958         while (defined(my $line = <$fh>)) {
6959             $new_conf_raw .= restore_update_config_line(
6960                 $cookie,
6961                 $map,
6962                 $line,
6963                 $opts->{unique},
6964             );
6965         }
6966
6967         $fh->close();
6968     };
6969
6970     eval {
6971         # enable interrupts
6972         local $SIG{INT} =
6973             local $SIG{TERM} =
6974             local $SIG{QUIT} =
6975             local $SIG{HUP} =
6976             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6977         local $SIG{ALRM} = sub { die "got timeout\n"; };
6978
6979         $oldtimeout = alarm($timeout);
6980
6981         my $parser = sub {
6982             my $line = shift;
6983
6984             print "$line\n";
6985
6986             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6987                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6988                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6989             } elsif ($line =~ m/^CTIME: /) {
6990                 # we correctly received the vma config, so we can disable
6991                 # the timeout now for disk allocation (set to 10 minutes, so
6992                 # that we always timeout if something goes wrong)
6993                 alarm(600);
6994                 &$print_devmap();
6995                 print $fifofh "done\n";
6996                 my $tmp = $oldtimeout || 0;
6997                 $oldtimeout = undef;
6998                 alarm($tmp);
6999                 close($fifofh);
7000                 $fifofh = undef;
7001             }
7002         };
7003
7004         print "restore vma archive: $dbg_cmdstring\n";
7005         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7006     };
7007     my $err = $@;
7008
7009     alarm($oldtimeout) if $oldtimeout;
7010
7011     $restore_deactivate_volumes->($cfg, $devinfo);
7012
7013     close($fifofh) if $fifofh;
7014     unlink $mapfifo;
7015     rmtree $tmpdir;
7016
7017     if ($err) {
7018         $restore_destroy_volumes->($cfg, $devinfo);
7019         die $err;
7020     }
7021
7022     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7023
7024     PVE::Cluster::cfs_update(); # make sure we read new file
7025
7026     eval { rescan($vmid, 1); };
7027     warn $@ if $@;
7028
7029     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7030 }
7031
7032 sub restore_tar_archive {
7033     my ($archive, $vmid, $user, $opts) = @_;
7034
7035     if ($archive ne '-') {
7036         my $firstfile = tar_archive_read_firstfile($archive);
7037         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7038             if $firstfile ne 'qemu-server.conf';
7039     }
7040
7041     my $storecfg = PVE::Storage::config();
7042
7043     # avoid zombie disks when restoring over an existing VM -> cleanup first
7044     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7045     # skiplock=1 because qmrestore has set the 'create' lock itself already
7046     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7047     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7048
7049     my $tocmd = "/usr/lib/qemu-server/qmextract";
7050
7051     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7052     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7053     $tocmd .= ' --prealloc' if $opts->{prealloc};
7054     $tocmd .= ' --info' if $opts->{info};
7055
7056     # tar option "xf" does not autodetect compression when read from STDIN,
7057     # so we pipe to zcat
7058     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7059         PVE::Tools::shellquote("--to-command=$tocmd");
7060
7061     my $tmpdir = "/var/tmp/vzdumptmp$$";
7062     mkpath $tmpdir;
7063
7064     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7065     local $ENV{VZDUMP_VMID} = $vmid;
7066     local $ENV{VZDUMP_USER} = $user;
7067
7068     my $conffile = PVE::QemuConfig->config_file($vmid);
7069     my $new_conf_raw = '';
7070
7071     # disable interrupts (always do cleanups)
7072     local $SIG{INT} =
7073         local $SIG{TERM} =
7074         local $SIG{QUIT} =
7075         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7076
7077     eval {
7078         # enable interrupts
7079         local $SIG{INT} =
7080             local $SIG{TERM} =
7081             local $SIG{QUIT} =
7082             local $SIG{HUP} =
7083             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7084
7085         if ($archive eq '-') {
7086             print "extracting archive from STDIN\n";
7087             run_command($cmd, input => "<&STDIN");
7088         } else {
7089             print "extracting archive '$archive'\n";
7090             run_command($cmd);
7091         }
7092
7093         return if $opts->{info};
7094
7095         # read new mapping
7096         my $map = {};
7097         my $statfile = "$tmpdir/qmrestore.stat";
7098         if (my $fd = IO::File->new($statfile, "r")) {
7099             while (defined (my $line = <$fd>)) {
7100                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7101                     $map->{$1} = $2 if $1;
7102                 } else {
7103                     print STDERR "unable to parse line in statfile - $line\n";
7104                 }
7105             }
7106             $fd->close();
7107         }
7108
7109         my $confsrc = "$tmpdir/qemu-server.conf";
7110
7111         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7112
7113         my $cookie = { netcount => 0 };
7114         while (defined (my $line = <$srcfd>)) {
7115             $new_conf_raw .= restore_update_config_line(
7116                 $cookie,
7117                 $map,
7118                 $line,
7119                 $opts->{unique},
7120             );
7121         }
7122
7123         $srcfd->close();
7124     };
7125     if (my $err = $@) {
7126         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7127         die $err;
7128     }
7129
7130     rmtree $tmpdir;
7131
7132     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7133
7134     PVE::Cluster::cfs_update(); # make sure we read new file
7135
7136     eval { rescan($vmid, 1); };
7137     warn $@ if $@;
7138 };
7139
7140 sub foreach_storage_used_by_vm {
7141     my ($conf, $func) = @_;
7142
7143     my $sidhash = {};
7144
7145     PVE::QemuConfig->foreach_volume($conf, sub {
7146         my ($ds, $drive) = @_;
7147         return if drive_is_cdrom($drive);
7148
7149         my $volid = $drive->{file};
7150
7151         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7152         $sidhash->{$sid} = $sid if $sid;
7153     });
7154
7155     foreach my $sid (sort keys %$sidhash) {
7156         &$func($sid);
7157     }
7158 }
7159
7160 my $qemu_snap_storage = {
7161     rbd => 1,
7162 };
7163 sub do_snapshots_with_qemu {
7164     my ($storecfg, $volid, $deviceid) = @_;
7165
7166     return if $deviceid =~ m/tpmstate0/;
7167
7168     my $storage_name = PVE::Storage::parse_volume_id($volid);
7169     my $scfg = $storecfg->{ids}->{$storage_name};
7170     die "could not find storage '$storage_name'\n" if !defined($scfg);
7171
7172     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7173         return 1;
7174     }
7175
7176     if ($volid =~ m/\.(qcow2|qed)$/){
7177         return 1;
7178     }
7179
7180     return;
7181 }
7182
7183 sub qga_check_running {
7184     my ($vmid, $nowarn) = @_;
7185
7186     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7187     if ($@) {
7188         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7189         return 0;
7190     }
7191     return 1;
7192 }
7193
7194 sub template_create {
7195     my ($vmid, $conf, $disk) = @_;
7196
7197     my $storecfg = PVE::Storage::config();
7198
7199     PVE::QemuConfig->foreach_volume($conf, sub {
7200         my ($ds, $drive) = @_;
7201
7202         return if drive_is_cdrom($drive);
7203         return if $disk && $ds ne $disk;
7204
7205         my $volid = $drive->{file};
7206         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7207
7208         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7209         $drive->{file} = $voliddst;
7210         $conf->{$ds} = print_drive($drive);
7211         PVE::QemuConfig->write_config($vmid, $conf);
7212     });
7213 }
7214
7215 sub convert_iscsi_path {
7216     my ($path) = @_;
7217
7218     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7219         my $portal = $1;
7220         my $target = $2;
7221         my $lun = $3;
7222
7223         my $initiator_name = get_initiator_name();
7224
7225         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7226                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7227     }
7228
7229     die "cannot convert iscsi path '$path', unkown format\n";
7230 }
7231
7232 sub qemu_img_convert {
7233     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7234
7235     my $storecfg = PVE::Storage::config();
7236     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7237     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7238
7239     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7240
7241     my $cachemode;
7242     my $src_path;
7243     my $src_is_iscsi = 0;
7244     my $src_format;
7245
7246     if ($src_storeid) {
7247         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7248         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7249         $src_format = qemu_img_format($src_scfg, $src_volname);
7250         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7251         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7252         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7253     } elsif (-f $src_volid) {
7254         $src_path = $src_volid;
7255         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7256             $src_format = $1;
7257         }
7258     }
7259
7260     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7261
7262     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7263     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7264     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7265     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7266
7267     my $cmd = [];
7268     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7269     push @$cmd, '-l', "snapshot.name=$snapname"
7270         if $snapname && $src_format && $src_format eq "qcow2";
7271     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7272     push @$cmd, '-T', $cachemode if defined($cachemode);
7273
7274     if ($src_is_iscsi) {
7275         push @$cmd, '--image-opts';
7276         $src_path = convert_iscsi_path($src_path);
7277     } elsif ($src_format) {
7278         push @$cmd, '-f', $src_format;
7279     }
7280
7281     if ($dst_is_iscsi) {
7282         push @$cmd, '--target-image-opts';
7283         $dst_path = convert_iscsi_path($dst_path);
7284     } else {
7285         push @$cmd, '-O', $dst_format;
7286     }
7287
7288     push @$cmd, $src_path;
7289
7290     if (!$dst_is_iscsi && $is_zero_initialized) {
7291         push @$cmd, "zeroinit:$dst_path";
7292     } else {
7293         push @$cmd, $dst_path;
7294     }
7295
7296     my $parser = sub {
7297         my $line = shift;
7298         if($line =~ m/\((\S+)\/100\%\)/){
7299             my $percent = $1;
7300             my $transferred = int($size * $percent / 100);
7301             my $total_h = render_bytes($size, 1);
7302             my $transferred_h = render_bytes($transferred, 1);
7303
7304             print "transferred $transferred_h of $total_h ($percent%)\n";
7305         }
7306
7307     };
7308
7309     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7310     my $err = $@;
7311     die "copy failed: $err" if $err;
7312 }
7313
7314 sub qemu_img_format {
7315     my ($scfg, $volname) = @_;
7316
7317     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7318         return $1;
7319     } else {
7320         return "raw";
7321     }
7322 }
7323
7324 sub qemu_drive_mirror {
7325     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7326
7327     $jobs = {} if !$jobs;
7328
7329     my $qemu_target;
7330     my $format;
7331     $jobs->{"drive-$drive"} = {};
7332
7333     if ($dst_volid =~ /^nbd:/) {
7334         $qemu_target = $dst_volid;
7335         $format = "nbd";
7336     } else {
7337         my $storecfg = PVE::Storage::config();
7338         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7339
7340         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7341
7342         $format = qemu_img_format($dst_scfg, $dst_volname);
7343
7344         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7345
7346         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7347     }
7348
7349     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7350     $opts->{format} = $format if $format;
7351
7352     if (defined($src_bitmap)) {
7353         $opts->{sync} = 'incremental';
7354         $opts->{bitmap} = $src_bitmap;
7355         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7356     }
7357
7358     if (defined($bwlimit)) {
7359         $opts->{speed} = $bwlimit * 1024;
7360         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7361     } else {
7362         print "drive mirror is starting for drive-$drive\n";
7363     }
7364
7365     # if a job already runs for this device we get an error, catch it for cleanup
7366     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7367     if (my $err = $@) {
7368         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7369         warn "$@\n" if $@;
7370         die "mirroring error: $err\n";
7371     }
7372
7373     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7374 }
7375
7376 # $completion can be either
7377 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7378 # 'cancel': wait until all jobs are ready, block-job-cancel them
7379 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7380 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7381 sub qemu_drive_mirror_monitor {
7382     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7383
7384     $completion //= 'complete';
7385     $op //= "mirror";
7386
7387     eval {
7388         my $err_complete = 0;
7389
7390         my $starttime = time ();
7391         while (1) {
7392             die "block job ('$op') timed out\n" if $err_complete > 300;
7393
7394             my $stats = mon_cmd($vmid, "query-block-jobs");
7395             my $ctime = time();
7396
7397             my $running_jobs = {};
7398             for my $stat (@$stats) {
7399                 next if $stat->{type} ne $op;
7400                 $running_jobs->{$stat->{device}} = $stat;
7401             }
7402
7403             my $readycounter = 0;
7404
7405             for my $job_id (sort keys %$jobs) {
7406                 my $job = $running_jobs->{$job_id};
7407
7408                 my $vanished = !defined($job);
7409                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7410                 if($complete || ($vanished && $completion eq 'auto')) {
7411                     print "$job_id: $op-job finished\n";
7412                     delete $jobs->{$job_id};
7413                     next;
7414                 }
7415
7416                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7417
7418                 my $busy = $job->{busy};
7419                 my $ready = $job->{ready};
7420                 if (my $total = $job->{len}) {
7421                     my $transferred = $job->{offset} || 0;
7422                     my $remaining = $total - $transferred;
7423                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7424
7425                     my $duration = $ctime - $starttime;
7426                     my $total_h = render_bytes($total, 1);
7427                     my $transferred_h = render_bytes($transferred, 1);
7428
7429                     my $status = sprintf(
7430                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7431                         render_duration($duration),
7432                     );
7433
7434                     if ($ready) {
7435                         if ($busy) {
7436                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7437                         } else {
7438                             $status .= ", ready";
7439                         }
7440                     }
7441                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7442                     $jobs->{$job_id}->{ready} = $ready;
7443                 }
7444
7445                 $readycounter++ if $job->{ready};
7446             }
7447
7448             last if scalar(keys %$jobs) == 0;
7449
7450             if ($readycounter == scalar(keys %$jobs)) {
7451                 print "all '$op' jobs are ready\n";
7452
7453                 # do the complete later (or has already been done)
7454                 last if $completion eq 'skip' || $completion eq 'auto';
7455
7456                 if ($vmiddst && $vmiddst != $vmid) {
7457                     my $agent_running = $qga && qga_check_running($vmid);
7458                     if ($agent_running) {
7459                         print "freeze filesystem\n";
7460                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7461                         warn $@ if $@;
7462                     } else {
7463                         print "suspend vm\n";
7464                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7465                         warn $@ if $@;
7466                     }
7467
7468                     # if we clone a disk for a new target vm, we don't switch the disk
7469                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7470
7471                     if ($agent_running) {
7472                         print "unfreeze filesystem\n";
7473                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7474                         warn $@ if $@;
7475                     } else {
7476                         print "resume vm\n";
7477                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7478                         warn $@ if $@;
7479                     }
7480
7481                     last;
7482                 } else {
7483
7484                     for my $job_id (sort keys %$jobs) {
7485                         # try to switch the disk if source and destination are on the same guest
7486                         print "$job_id: Completing block job_id...\n";
7487
7488                         my $op;
7489                         if ($completion eq 'complete') {
7490                             $op = 'block-job-complete';
7491                         } elsif ($completion eq 'cancel') {
7492                             $op = 'block-job-cancel';
7493                         } else {
7494                             die "invalid completion value: $completion\n";
7495                         }
7496                         eval { mon_cmd($vmid, $op, device => $job_id) };
7497                         if ($@ =~ m/cannot be completed/) {
7498                             print "$job_id: block job cannot be completed, trying again.\n";
7499                             $err_complete++;
7500                         }else {
7501                             print "$job_id: Completed successfully.\n";
7502                             $jobs->{$job_id}->{complete} = 1;
7503                         }
7504                     }
7505                 }
7506             }
7507             sleep 1;
7508         }
7509     };
7510     my $err = $@;
7511
7512     if ($err) {
7513         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7514         die "block job ($op) error: $err";
7515     }
7516 }
7517
7518 sub qemu_blockjobs_cancel {
7519     my ($vmid, $jobs) = @_;
7520
7521     foreach my $job (keys %$jobs) {
7522         print "$job: Cancelling block job\n";
7523         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7524         $jobs->{$job}->{cancel} = 1;
7525     }
7526
7527     while (1) {
7528         my $stats = mon_cmd($vmid, "query-block-jobs");
7529
7530         my $running_jobs = {};
7531         foreach my $stat (@$stats) {
7532             $running_jobs->{$stat->{device}} = $stat;
7533         }
7534
7535         foreach my $job (keys %$jobs) {
7536
7537             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7538                 print "$job: Done.\n";
7539                 delete $jobs->{$job};
7540             }
7541         }
7542
7543         last if scalar(keys %$jobs) == 0;
7544
7545         sleep 1;
7546     }
7547 }
7548
7549 sub clone_disk {
7550     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7551         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7552
7553     my $newvolid;
7554
7555     if (!$full) {
7556         print "create linked clone of drive $drivename ($drive->{file})\n";
7557         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7558         push @$newvollist, $newvolid;
7559     } else {
7560
7561         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7562         $storeid = $storage if $storage;
7563
7564         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7565
7566         print "create full clone of drive $drivename ($drive->{file})\n";
7567         my $name = undef;
7568         my $size = undef;
7569         if (drive_is_cloudinit($drive)) {
7570             $name = "vm-$newvmid-cloudinit";
7571             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7572             if ($scfg->{path}) {
7573                 $name .= ".$dst_format";
7574             }
7575             $snapname = undef;
7576             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7577         } elsif ($drivename eq 'efidisk0') {
7578             $size = get_efivars_size($conf);
7579         } elsif ($drivename eq 'tpmstate0') {
7580             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7581         } else {
7582             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7583         }
7584         $newvolid = PVE::Storage::vdisk_alloc(
7585             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7586         );
7587         push @$newvollist, $newvolid;
7588
7589         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7590
7591         if (drive_is_cloudinit($drive)) {
7592             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7593             # if this is the case, we have to complete any block-jobs still there from
7594             # previous drive-mirrors
7595             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7596                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7597             }
7598             goto no_data_clone;
7599         }
7600
7601         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7602         if (!$running || $snapname) {
7603             # TODO: handle bwlimits
7604             if ($drivename eq 'efidisk0') {
7605                 # the relevant data on the efidisk may be smaller than the source
7606                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7607                 # that is given by the OVMF_VARS.fd
7608                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7609                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7610
7611                 # better for Ceph if block size is not too small, see bug #3324
7612                 my $bs = 1024*1024;
7613
7614                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7615                     "if=$src_path", "of=$dst_path"]);
7616             } else {
7617                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7618             }
7619         } else {
7620
7621             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7622
7623             my $kvmver = get_running_qemu_version ($vmid);
7624             if (!min_version($kvmver, 2, 7)) {
7625                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7626                     if $drive->{iothread};
7627             }
7628
7629             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7630                 $completion, $qga, $bwlimit);
7631         }
7632     }
7633
7634 no_data_clone:
7635     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7636
7637     my $disk = dclone($drive);
7638     delete $disk->{format};
7639     $disk->{file} = $newvolid;
7640     $disk->{size} = $size if defined($size);
7641
7642     return $disk;
7643 }
7644
7645 sub get_running_qemu_version {
7646     my ($vmid) = @_;
7647     my $res = mon_cmd($vmid, "query-version");
7648     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7649 }
7650
7651 sub qemu_use_old_bios_files {
7652     my ($machine_type) = @_;
7653
7654     return if !$machine_type;
7655
7656     my $use_old_bios_files = undef;
7657
7658     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7659         $machine_type = $1;
7660         $use_old_bios_files = 1;
7661     } else {
7662         my $version = extract_version($machine_type, kvm_user_version());
7663         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7664         # load new efi bios files on migration. So this hack is required to allow
7665         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7666         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7667         $use_old_bios_files = !min_version($version, 2, 4);
7668     }
7669
7670     return ($use_old_bios_files, $machine_type);
7671 }
7672
7673 sub get_efivars_size {
7674     my ($conf) = @_;
7675     my $arch = get_vm_arch($conf);
7676     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7677     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7678     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7679     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7680     return -s $ovmf_vars;
7681 }
7682
7683 sub update_efidisk_size {
7684     my ($conf) = @_;
7685
7686     return if !defined($conf->{efidisk0});
7687
7688     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7689     $disk->{size} = get_efivars_size($conf);
7690     $conf->{efidisk0} = print_drive($disk);
7691
7692     return;
7693 }
7694
7695 sub update_tpmstate_size {
7696     my ($conf) = @_;
7697
7698     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7699     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7700     $conf->{tpmstate0} = print_drive($disk);
7701 }
7702
7703 sub create_efidisk($$$$$$$) {
7704     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7705
7706     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7707     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7708
7709     my $vars_size_b = -s $ovmf_vars;
7710     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7711     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7712     PVE::Storage::activate_volumes($storecfg, [$volid]);
7713
7714     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7715     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7716
7717     return ($volid, $size/1024);
7718 }
7719
7720 sub vm_iothreads_list {
7721     my ($vmid) = @_;
7722
7723     my $res = mon_cmd($vmid, 'query-iothreads');
7724
7725     my $iothreads = {};
7726     foreach my $iothread (@$res) {
7727         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7728     }
7729
7730     return $iothreads;
7731 }
7732
7733 sub scsihw_infos {
7734     my ($conf, $drive) = @_;
7735
7736     my $maxdev = 0;
7737
7738     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7739         $maxdev = 7;
7740     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7741         $maxdev = 1;
7742     } else {
7743         $maxdev = 256;
7744     }
7745
7746     my $controller = int($drive->{index} / $maxdev);
7747     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7748         ? "virtioscsi"
7749         : "scsihw";
7750
7751     return ($maxdev, $controller, $controller_prefix);
7752 }
7753
7754 sub windows_version {
7755     my ($ostype) = @_;
7756
7757     return 0 if !$ostype;
7758
7759     my $winversion = 0;
7760
7761     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7762         $winversion = 5;
7763     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7764         $winversion = 6;
7765     } elsif ($ostype =~ m/^win(\d+)$/) {
7766         $winversion = $1;
7767     }
7768
7769     return $winversion;
7770 }
7771
7772 sub resolve_dst_disk_format {
7773         my ($storecfg, $storeid, $src_volname, $format) = @_;
7774         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7775
7776         if (!$format) {
7777             # if no target format is specified, use the source disk format as hint
7778             if ($src_volname) {
7779                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7780                 $format = qemu_img_format($scfg, $src_volname);
7781             } else {
7782                 return $defFormat;
7783             }
7784         }
7785
7786         # test if requested format is supported - else use default
7787         my $supported = grep { $_ eq $format } @$validFormats;
7788         $format = $defFormat if !$supported;
7789         return $format;
7790 }
7791
7792 # NOTE: if this logic changes, please update docs & possibly gui logic
7793 sub find_vmstate_storage {
7794     my ($conf, $storecfg) = @_;
7795
7796     # first, return storage from conf if set
7797     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7798
7799     my ($target, $shared, $local);
7800
7801     foreach_storage_used_by_vm($conf, sub {
7802         my ($sid) = @_;
7803         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7804         my $dst = $scfg->{shared} ? \$shared : \$local;
7805         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7806     });
7807
7808     # second, use shared storage where VM has at least one disk
7809     # third, use local storage where VM has at least one disk
7810     # fall back to local storage
7811     $target = $shared // $local // 'local';
7812
7813     return $target;
7814 }
7815
7816 sub generate_uuid {
7817     my ($uuid, $uuid_str);
7818     UUID::generate($uuid);
7819     UUID::unparse($uuid, $uuid_str);
7820     return $uuid_str;
7821 }
7822
7823 sub generate_smbios1_uuid {
7824     return "uuid=".generate_uuid();
7825 }
7826
7827 sub nbd_stop {
7828     my ($vmid) = @_;
7829
7830     mon_cmd($vmid, 'nbd-server-stop');
7831 }
7832
7833 sub create_reboot_request {
7834     my ($vmid) = @_;
7835     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7836         or die "failed to create reboot trigger file: $!\n";
7837     close($fh);
7838 }
7839
7840 sub clear_reboot_request {
7841     my ($vmid) = @_;
7842     my $path = "/run/qemu-server/$vmid.reboot";
7843     my $res = 0;
7844
7845     $res = unlink($path);
7846     die "could not remove reboot request for $vmid: $!"
7847         if !$res && $! != POSIX::ENOENT;
7848
7849     return $res;
7850 }
7851
7852 sub bootorder_from_legacy {
7853     my ($conf, $bootcfg) = @_;
7854
7855     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7856     my $bootindex_hash = {};
7857     my $i = 1;
7858     foreach my $o (split(//, $boot)) {
7859         $bootindex_hash->{$o} = $i*100;
7860         $i++;
7861     }
7862
7863     my $bootorder = {};
7864
7865     PVE::QemuConfig->foreach_volume($conf, sub {
7866         my ($ds, $drive) = @_;
7867
7868         if (drive_is_cdrom ($drive, 1)) {
7869             if ($bootindex_hash->{d}) {
7870                 $bootorder->{$ds} = $bootindex_hash->{d};
7871                 $bootindex_hash->{d} += 1;
7872             }
7873         } elsif ($bootindex_hash->{c}) {
7874             $bootorder->{$ds} = $bootindex_hash->{c}
7875                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7876             $bootindex_hash->{c} += 1;
7877         }
7878     });
7879
7880     if ($bootindex_hash->{n}) {
7881         for (my $i = 0; $i < $MAX_NETS; $i++) {
7882             my $netname = "net$i";
7883             next if !$conf->{$netname};
7884             $bootorder->{$netname} = $bootindex_hash->{n};
7885             $bootindex_hash->{n} += 1;
7886         }
7887     }
7888
7889     return $bootorder;
7890 }
7891
7892 # Generate default device list for 'boot: order=' property. Matches legacy
7893 # default boot order, but with explicit device names. This is important, since
7894 # the fallback for when neither 'order' nor the old format is specified relies
7895 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7896 sub get_default_bootdevices {
7897     my ($conf) = @_;
7898
7899     my @ret = ();
7900
7901     # harddisk
7902     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7903     push @ret, $first if $first;
7904
7905     # cdrom
7906     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7907     push @ret, $first if $first;
7908
7909     # network
7910     for (my $i = 0; $i < $MAX_NETS; $i++) {
7911         my $netname = "net$i";
7912         next if !$conf->{$netname};
7913         push @ret, $netname;
7914         last;
7915     }
7916
7917     return \@ret;
7918 }
7919
7920 sub device_bootorder {
7921     my ($conf) = @_;
7922
7923     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7924
7925     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7926
7927     my $bootorder = {};
7928     if (!defined($boot) || $boot->{legacy}) {
7929         $bootorder = bootorder_from_legacy($conf, $boot);
7930     } elsif ($boot->{order}) {
7931         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7932         for my $dev (PVE::Tools::split_list($boot->{order})) {
7933             $bootorder->{$dev} = $i++;
7934         }
7935     }
7936
7937     return $bootorder;
7938 }
7939
7940 sub register_qmeventd_handle {
7941     my ($vmid) = @_;
7942
7943     my $fh;
7944     my $peer = "/var/run/qmeventd.sock";
7945     my $count = 0;
7946
7947     for (;;) {
7948         $count++;
7949         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7950         last if $fh;
7951         if ($! != EINTR && $! != EAGAIN) {
7952             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7953         }
7954         if ($count > 4) {
7955             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7956               . "after $count retries\n";
7957         }
7958         usleep(25000);
7959     }
7960
7961     # send handshake to mark VM as backing up
7962     print $fh to_json({vzdump => {vmid => "$vmid"}});
7963
7964     # return handle to be closed later when inhibit is no longer required
7965     return $fh;
7966 }
7967
7968 # bash completion helper
7969
7970 sub complete_backup_archives {
7971     my ($cmdname, $pname, $cvalue) = @_;
7972
7973     my $cfg = PVE::Storage::config();
7974
7975     my $storeid;
7976
7977     if ($cvalue =~ m/^([^:]+):/) {
7978         $storeid = $1;
7979     }
7980
7981     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7982
7983     my $res = [];
7984     foreach my $id (keys %$data) {
7985         foreach my $item (@{$data->{$id}}) {
7986             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7987             push @$res, $item->{volid} if defined($item->{volid});
7988         }
7989     }
7990
7991     return $res;
7992 }
7993
7994 my $complete_vmid_full = sub {
7995     my ($running) = @_;
7996
7997     my $idlist = vmstatus();
7998
7999     my $res = [];
8000
8001     foreach my $id (keys %$idlist) {
8002         my $d = $idlist->{$id};
8003         if (defined($running)) {
8004             next if $d->{template};
8005             next if $running && $d->{status} ne 'running';
8006             next if !$running && $d->{status} eq 'running';
8007         }
8008         push @$res, $id;
8009
8010     }
8011     return $res;
8012 };
8013
8014 sub complete_vmid {
8015     return &$complete_vmid_full();
8016 }
8017
8018 sub complete_vmid_stopped {
8019     return &$complete_vmid_full(0);
8020 }
8021
8022 sub complete_vmid_running {
8023     return &$complete_vmid_full(1);
8024 }
8025
8026 sub complete_storage {
8027
8028     my $cfg = PVE::Storage::config();
8029     my $ids = $cfg->{ids};
8030
8031     my $res = [];
8032     foreach my $sid (keys %$ids) {
8033         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8034         next if !$ids->{$sid}->{content}->{images};
8035         push @$res, $sid;
8036     }
8037
8038     return $res;
8039 }
8040
8041 sub complete_migration_storage {
8042     my ($cmd, $param, $current_value, $all_args) = @_;
8043
8044     my $targetnode = @$all_args[1];
8045
8046     my $cfg = PVE::Storage::config();
8047     my $ids = $cfg->{ids};
8048
8049     my $res = [];
8050     foreach my $sid (keys %$ids) {
8051         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8052         next if !$ids->{$sid}->{content}->{images};
8053         push @$res, $sid;
8054     }
8055
8056     return $res;
8057 }
8058
8059 sub vm_is_paused {
8060     my ($vmid) = @_;
8061     my $qmpstatus = eval {
8062         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8063         mon_cmd($vmid, "query-status");
8064     };
8065     warn "$@\n" if $@;
8066     return $qmpstatus && $qmpstatus->{status} eq "paused";
8067 }
8068
8069 sub check_volume_storage_type {
8070     my ($storecfg, $vol) = @_;
8071
8072     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8073     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8074     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8075
8076     die "storage '$storeid' does not support content-type '$vtype'\n"
8077         if !$scfg->{content}->{$vtype};
8078
8079     return 1;
8080 }
8081
8082 1;