PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122
 123 sub map_storage {
 124     my ($map, $source) = @_;
 125
 126     return $source if !defined($map);
 127
 128     return $map->{entries}->{$source}
 129         if $map->{entries} && defined($map->{entries}->{$source});
 130
 131     return $map->{default} if $map->{default};
 132
 133     # identity (fallback)
 134     return $source;
 135 }
 136
 137 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 138     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 139     type => 'string',
 140     format => 'storagepair-list',
 141     optional => 1,
 142 });
 143
 144 #no warnings 'redefine';
 145
 146 my $nodename_cache;
 147 sub nodename {
 148     $nodename_cache //= PVE::INotify::nodename();
 149     return $nodename_cache;
 150 }
 151
 152 my $watchdog_fmt = {
 153     model => {
 154         default_key => 1,
 155         type => 'string',
 156         enum => [qw(i6300esb ib700)],
 157         description => "Watchdog type to emulate.",
 158         default => 'i6300esb',
 159         optional => 1,
 160     },
 161     action => {
 162         type => 'string',
 163         enum => [qw(reset shutdown poweroff pause debug none)],
 164         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 165         optional => 1,
 166     },
 167 };
 168 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 169
 170 my $agent_fmt = {
 171     enabled => {
 172         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 173         type => 'boolean',
 174         default => 0,
 175         default_key => 1,
 176     },
 177     fstrim_cloned_disks => {
 178         description => "Run fstrim after moving a disk or migrating the VM.",
 179         type => 'boolean',
 180         optional => 1,
 181         default => 0
 182     },
 183     type => {
 184         description => "Select the agent type",
 185         type => 'string',
 186         default => 'virtio',
 187         optional => 1,
 188         enum => [qw(virtio isa)],
 189     },
 190 };
 191
 192 my $vga_fmt = {
 193     type => {
 194         description => "Select the VGA type.",
 195         type => 'string',
 196         default => 'std',
 197         optional => 1,
 198         default_key => 1,
 199         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 200     },
 201     memory => {
 202         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 203         type => 'integer',
 204         optional => 1,
 205         minimum => 4,
 206         maximum => 512,
 207     },
 208 };
 209
 210 my $ivshmem_fmt = {
 211     size => {
 212         type => 'integer',
 213         minimum => 1,
 214         description => "The size of the file in MB.",
 215     },
 216     name => {
 217         type => 'string',
 218         pattern => '[a-zA-Z0-9\-]+',
 219         optional => 1,
 220         format_description => 'string',
 221         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 222     },
 223 };
 224
 225 my $audio_fmt = {
 226     device => {
 227         type => 'string',
 228         enum => [qw(ich9-intel-hda intel-hda AC97)],
 229         description =>  "Configure an audio device."
 230     },
 231     driver =>  {
 232         type => 'string',
 233         enum => ['spice', 'none'],
 234         default => 'spice',
 235         optional => 1,
 236         description => "Driver backend for the audio device."
 237     },
 238 };
 239
 240 my $spice_enhancements_fmt = {
 241     foldersharing => {
 242         type => 'boolean',
 243         optional => 1,
 244         default => '0',
 245         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 246     },
 247     videostreaming =>  {
 248         type => 'string',
 249         enum => ['off', 'all', 'filter'],
 250         default => 'off',
 251         optional => 1,
 252         description => "Enable video streaming. Uses compression for detected video streams."
 253     },
 254 };
 255
 256 my $rng_fmt = {
 257     source => {
 258         type => 'string',
 259         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 260         default_key => 1,
 261         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 262             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 263             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 264             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 265             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 266             ." a hardware RNG from the host.",
 267     },
 268     max_bytes => {
 269         type => 'integer',
 270         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 271             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 272             ." `0` to disable limiting (potentially dangerous!).",
 273         optional => 1,
 274
 275         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 276         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 277         # reading from /dev/urandom
 278         default => 1024,
 279     },
 280     period => {
 281         type => 'integer',
 282         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 283             ." the guest to retrieve another 'max_bytes' of entropy.",
 284         optional => 1,
 285         default => 1000,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 2,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. The default is read from the"
 371             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710 };
 711
 712 my $cicustom_fmt = {
 713     meta => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all meta data passed to the VM via"
 717             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     network => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all network data passed to the VM via'
 725             .' cloud-init.',
 726         format => 'pve-volume-id',
 727         format_description => 'volume',
 728     },
 729     user => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all user data passed to the VM via'
 733             .' cloud-init.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737 };
 738 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 739
 740 my $confdesc_cloudinit = {
 741     citype => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'Specifies the cloud-init configuration format. The default depends on the'
 745             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 746             .' and `configdrive2` for windows.',
 747         enum => ['configdrive2', 'nocloud', 'opennebula'],
 748     },
 749     ciuser => {
 750         optional => 1,
 751         type => 'string',
 752         description => "cloud-init: User name to change ssh keys and password for instead of the"
 753             ." image's configured default user.",
 754     },
 755     cipassword => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'cloud-init: Password to assign the user. Using this is generally not'
 759             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 760             .' support hashed passwords.',
 761     },
 762     cicustom => {
 763         optional => 1,
 764         type => 'string',
 765         description => 'cloud-init: Specify custom files to replace the automatically generated'
 766             .' ones at start.',
 767         format => 'pve-qm-cicustom',
 768     },
 769     searchdomain => {
 770         optional => 1,
 771         type => 'string',
 772         description => "cloud-init: Sets DNS search domains for a container. Create will'
 773             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 774             .' are set.",
 775     },
 776     nameserver => {
 777         optional => 1,
 778         type => 'string', format => 'address-list',
 779         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 780             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 781             .' are set.",
 782     },
 783     sshkeys => {
 784         optional => 1,
 785         type => 'string',
 786         format => 'urlencoded',
 787         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 788     },
 789 };
 790
 791 # what about other qemu settings ?
 792 #cpu => 'string',
 793 #machine => 'string',
 794 #fda => 'file',
 795 #fdb => 'file',
 796 #mtdblock => 'file',
 797 #sd => 'file',
 798 #pflash => 'file',
 799 #snapshot => 'bool',
 800 #bootp => 'file',
 801 ##tftp => 'dir',
 802 ##smb => 'dir',
 803 #kernel => 'file',
 804 #append => 'string',
 805 #initrd => 'file',
 806 ##soundhw => 'string',
 807
 808 while (my ($k, $v) = each %$confdesc) {
 809     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 810 }
 811
 812 my $MAX_USB_DEVICES = 5;
 813 my $MAX_NETS = 32;
 814 my $MAX_SERIAL_PORTS = 4;
 815 my $MAX_PARALLEL_PORTS = 3;
 816 my $MAX_NUMA = 8;
 817
 818 my $numa_fmt = {
 819     cpus => {
 820         type => "string",
 821         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 822         description => "CPUs accessing this NUMA node.",
 823         format_description => "id[-id];...",
 824     },
 825     memory => {
 826         type => "number",
 827         description => "Amount of memory this NUMA node provides.",
 828         optional => 1,
 829     },
 830     hostnodes => {
 831         type => "string",
 832         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 833         description => "Host NUMA nodes to use.",
 834         format_description => "id[-id];...",
 835         optional => 1,
 836     },
 837     policy => {
 838         type => 'string',
 839         enum => [qw(preferred bind interleave)],
 840         description => "NUMA allocation policy.",
 841         optional => 1,
 842     },
 843 };
 844 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 845 my $numadesc = {
 846     optional => 1,
 847     type => 'string', format => $numa_fmt,
 848     description => "NUMA topology.",
 849 };
 850 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 851
 852 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 853     $confdesc->{"numa$i"} = $numadesc;
 854 }
 855
 856 my $nic_model_list = [
 857     'e1000',
 858     'e1000-82540em',
 859     'e1000-82544gc',
 860     'e1000-82545em',
 861     'e1000e',
 862     'i82551',
 863     'i82557b',
 864     'i82559er',
 865     'ne2k_isa',
 866     'ne2k_pci',
 867     'pcnet',
 868     'rtl8139',
 869     'virtio',
 870     'vmxnet3',
 871 ];
 872 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 873
 874 my $net_fmt_bridge_descr = <<__EOD__;
 875 Bridge to attach the network device to. The Proxmox VE standard bridge
 876 is called 'vmbr0'.
 877
 878 If you do not specify a bridge, we create a kvm user (NATed) network
 879 device, which provides DHCP and DNS services. The following addresses
 880 are used:
 881
 882  10.0.2.2   Gateway
 883  10.0.2.3   DNS Server
 884  10.0.2.4   SMB Server
 885
 886 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 887 __EOD__
 888
 889 my $net_fmt = {
 890     macaddr  => get_standard_option('mac-addr', {
 891         description => "MAC address. That address must be unique withing your network. This is"
 892             ." automatically generated if not specified.",
 893     }),
 894     model => {
 895         type => 'string',
 896         description => "Network Card Model. The 'virtio' model provides the best performance with"
 897             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 898             ." best to use 'e1000'.",
 899         enum => $nic_model_list,
 900         default_key => 1,
 901     },
 902     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 903     bridge => {
 904         type => 'string',
 905         description => $net_fmt_bridge_descr,
 906         format_description => 'bridge',
 907         pattern => '[-_.\w\d]+',
 908         optional => 1,
 909     },
 910     queues => {
 911         type => 'integer',
 912         minimum => 0, maximum => 16,
 913         description => 'Number of packet queues to be used on the device.',
 914         optional => 1,
 915     },
 916     rate => {
 917         type => 'number',
 918         minimum => 0,
 919         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 920         optional => 1,
 921     },
 922     tag => {
 923         type => 'integer',
 924         minimum => 1, maximum => 4094,
 925         description => 'VLAN tag to apply to packets on this interface.',
 926         optional => 1,
 927     },
 928     trunks => {
 929         type => 'string',
 930         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 931         description => 'VLAN trunks to pass through this interface.',
 932         format_description => 'vlanid[;vlanid...]',
 933         optional => 1,
 934     },
 935     firewall => {
 936         type => 'boolean',
 937         description => 'Whether this interface should be protected by the firewall.',
 938         optional => 1,
 939     },
 940     link_down => {
 941         type => 'boolean',
 942         description => 'Whether this interface should be disconnected (like pulling the plug).',
 943         optional => 1,
 944     },
 945     mtu => {
 946         type => 'integer',
 947         minimum => 1, maximum => 65520,
 948         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 949         optional => 1,
 950     },
 951 };
 952
 953 my $netdesc = {
 954     optional => 1,
 955     type => 'string', format => $net_fmt,
 956     description => "Specify network devices.",
 957 };
 958
 959 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 960
 961 my $ipconfig_fmt = {
 962     ip => {
 963         type => 'string',
 964         format => 'pve-ipv4-config',
 965         format_description => 'IPv4Format/CIDR',
 966         description => 'IPv4 address in CIDR format.',
 967         optional => 1,
 968         default => 'dhcp',
 969     },
 970     gw => {
 971         type => 'string',
 972         format => 'ipv4',
 973         format_description => 'GatewayIPv4',
 974         description => 'Default gateway for IPv4 traffic.',
 975         optional => 1,
 976         requires => 'ip',
 977     },
 978     ip6 => {
 979         type => 'string',
 980         format => 'pve-ipv6-config',
 981         format_description => 'IPv6Format/CIDR',
 982         description => 'IPv6 address in CIDR format.',
 983         optional => 1,
 984         default => 'dhcp',
 985     },
 986     gw6 => {
 987         type => 'string',
 988         format => 'ipv6',
 989         format_description => 'GatewayIPv6',
 990         description => 'Default gateway for IPv6 traffic.',
 991         optional => 1,
 992         requires => 'ip6',
 993     },
 994 };
 995 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 996 my $ipconfigdesc = {
 997     optional => 1,
 998     type => 'string', format => 'pve-qm-ipconfig',
 999     description => <<'EODESCR',
1000 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1001
1002 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1003
1004 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1005 gateway should be provided.
1006 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1007 cloud-init 19.4 or newer.
1008
1009 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1010 dhcp on IPv4.
1011 EODESCR
1012 };
1013 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1014
1015 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1016     $confdesc->{"net$i"} = $netdesc;
1017     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1018 }
1019
1020 foreach my $key (keys %$confdesc_cloudinit) {
1021     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1022 }
1023
1024 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1025 sub verify_volume_id_or_qm_path {
1026     my ($volid, $noerr) = @_;
1027
1028     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1029         return $volid;
1030     }
1031
1032     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1033     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1034     if ($@) {
1035         return if $noerr;
1036         die $@;
1037     }
1038     return $volid;
1039 }
1040
1041 my $usb_fmt = {
1042     host => {
1043         default_key => 1,
1044         type => 'string', format => 'pve-qm-usb-device',
1045         format_description => 'HOSTUSBDEVICE|spice',
1046         description => <<EODESCR,
1047 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1048
1049  'bus-port(.port)*' (decimal numbers) or
1050  'vendor_id:product_id' (hexadeciaml numbers) or
1051  'spice'
1052
1053 You can use the 'lsusb -t' command to list existing usb devices.
1054
1055 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1056 machines - use with special care.
1057
1058 The value 'spice' can be used to add a usb redirection devices for spice.
1059 EODESCR
1060     },
1061     usb3 => {
1062         optional => 1,
1063         type => 'boolean',
1064         description => "Specifies whether if given host option is a USB3 device or port.",
1065         default => 0,
1066     },
1067 };
1068
1069 my $usbdesc = {
1070     optional => 1,
1071     type => 'string', format => $usb_fmt,
1072     description => "Configure an USB device (n is 0 to 4).",
1073 };
1074 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1075
1076 my $serialdesc = {
1077         optional => 1,
1078         type => 'string',
1079         pattern => '(/dev/.+|socket)',
1080         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1081         verbose_description =>  <<EODESCR,
1082 Create a serial device inside the VM (n is 0 to 3), and pass through a
1083 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1084 host side (use 'qm terminal' to open a terminal connection).
1085
1086 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1087 use with special care.
1088
1089 CAUTION: Experimental! User reported problems with this option.
1090 EODESCR
1091 };
1092
1093 my $paralleldesc= {
1094         optional => 1,
1095         type => 'string',
1096         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1097         description =>  "Map host parallel devices (n is 0 to 2).",
1098         verbose_description =>  <<EODESCR,
1099 Map host parallel devices (n is 0 to 2).
1100
1101 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1102 machines - use with special care.
1103
1104 CAUTION: Experimental! User reported problems with this option.
1105 EODESCR
1106 };
1107
1108 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1109     $confdesc->{"parallel$i"} = $paralleldesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1113     $confdesc->{"serial$i"} = $serialdesc;
1114 }
1115
1116 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1117     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1118 }
1119
1120 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1121     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1122 }
1123
1124 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1125     $confdesc->{"usb$i"} = $usbdesc;
1126 }
1127
1128 my $boot_fmt = {
1129     legacy => {
1130         optional => 1,
1131         default_key => 1,
1132         type => 'string',
1133         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1134                      . " Deprecated, use 'order=' instead.",
1135         pattern => '[acdn]{1,4}',
1136         format_description => "[acdn]{1,4}",
1137
1138         # note: this is also the fallback if boot: is not given at all
1139         default => 'cdn',
1140     },
1141     order => {
1142         optional => 1,
1143         type => 'string',
1144         format => 'pve-qm-bootdev-list',
1145         format_description => "device[;device...]",
1146         description => <<EODESC,
1147 The guest will attempt to boot from devices in the order they appear here.
1148
1149 Disks, optical drives and passed-through storage USB devices will be directly
1150 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1151 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1152
1153 Note that only devices in this list will be marked as bootable and thus loaded
1154 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1155 (e.g. software-raid), you need to specify all of them here.
1156
1157 Overrides the deprecated 'legacy=[acdn]*' value when given.
1158 EODESC
1159     },
1160 };
1161 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1162
1163 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1164 sub verify_bootdev {
1165     my ($dev, $noerr) = @_;
1166
1167     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1168     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1169
1170     my $check = sub {
1171         my ($base) = @_;
1172         return 0 if $dev !~ m/^$base\d+$/;
1173         return 0 if !$confdesc->{$dev};
1174         return 1;
1175     };
1176
1177     return $dev if $check->("net");
1178     return $dev if $check->("usb");
1179     return $dev if $check->("hostpci");
1180
1181     return if $noerr;
1182     die "invalid boot device '$dev'\n";
1183 }
1184
1185 sub print_bootorder {
1186     my ($devs) = @_;
1187     return "" if !@$devs;
1188     my $data = { order => join(';', @$devs) };
1189     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1190 }
1191
1192 my $kvm_api_version = 0;
1193
1194 sub kvm_version {
1195     return $kvm_api_version if $kvm_api_version;
1196
1197     open my $fh, '<', '/dev/kvm' or return;
1198
1199     # 0xae00 => KVM_GET_API_VERSION
1200     $kvm_api_version = ioctl($fh, 0xae00, 0);
1201     close($fh);
1202
1203     return $kvm_api_version;
1204 }
1205
1206 my $kvm_user_version = {};
1207 my $kvm_mtime = {};
1208
1209 sub kvm_user_version {
1210     my ($binary) = @_;
1211
1212     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1213     my $st = stat($binary);
1214
1215     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1216     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1217         $cachedmtime == $st->mtime;
1218
1219     $kvm_user_version->{$binary} = 'unknown';
1220     $kvm_mtime->{$binary} = $st->mtime;
1221
1222     my $code = sub {
1223         my $line = shift;
1224         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1225             $kvm_user_version->{$binary} = $2;
1226         }
1227     };
1228
1229     eval { run_command([$binary, '--version'], outfunc => $code); };
1230     warn $@ if $@;
1231
1232     return $kvm_user_version->{$binary};
1233
1234 }
1235 my sub extract_version {
1236     my ($machine_type, $version) = @_;
1237     $version = kvm_user_version() if !defined($version);
1238     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1239 }
1240
1241 sub kernel_has_vhost_net {
1242     return -c '/dev/vhost-net';
1243 }
1244
1245 sub option_exists {
1246     my $key = shift;
1247     return defined($confdesc->{$key});
1248 }
1249
1250 my $cdrom_path;
1251 sub get_cdrom_path {
1252
1253     return  $cdrom_path if $cdrom_path;
1254
1255     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1256     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1257     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1258 }
1259
1260 sub get_iso_path {
1261     my ($storecfg, $vmid, $cdrom) = @_;
1262
1263     if ($cdrom eq 'cdrom') {
1264         return get_cdrom_path();
1265     } elsif ($cdrom eq 'none') {
1266         return '';
1267     } elsif ($cdrom =~ m|^/|) {
1268         return $cdrom;
1269     } else {
1270         return PVE::Storage::path($storecfg, $cdrom);
1271     }
1272 }
1273
1274 # try to convert old style file names to volume IDs
1275 sub filename_to_volume_id {
1276     my ($vmid, $file, $media) = @_;
1277
1278      if (!($file eq 'none' || $file eq 'cdrom' ||
1279           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1280
1281         return if $file =~ m|/|;
1282
1283         if ($media && $media eq 'cdrom') {
1284             $file = "local:iso/$file";
1285         } else {
1286             $file = "local:$vmid/$file";
1287         }
1288     }
1289
1290     return $file;
1291 }
1292
1293 sub verify_media_type {
1294     my ($opt, $vtype, $media) = @_;
1295
1296     return if !$media;
1297
1298     my $etype;
1299     if ($media eq 'disk') {
1300         $etype = 'images';
1301     } elsif ($media eq 'cdrom') {
1302         $etype = 'iso';
1303     } else {
1304         die "internal error";
1305     }
1306
1307     return if ($vtype eq $etype);
1308
1309     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1310 }
1311
1312 sub cleanup_drive_path {
1313     my ($opt, $storecfg, $drive) = @_;
1314
1315     # try to convert filesystem paths to volume IDs
1316
1317     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1318         ($drive->{file} !~ m|^/dev/.+|) &&
1319         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1320         ($drive->{file} !~ m/^\d+$/)) {
1321         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1322         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1323             if !$vtype;
1324         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1325         verify_media_type($opt, $vtype, $drive->{media});
1326         $drive->{file} = $volid;
1327     }
1328
1329     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1330 }
1331
1332 sub parse_hotplug_features {
1333     my ($data) = @_;
1334
1335     my $res = {};
1336
1337     return $res if $data eq '0';
1338
1339     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1340
1341     foreach my $feature (PVE::Tools::split_list($data)) {
1342         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1343             $res->{$1} = 1;
1344         } else {
1345             die "invalid hotplug feature '$feature'\n";
1346         }
1347     }
1348     return $res;
1349 }
1350
1351 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1352 sub pve_verify_hotplug_features {
1353     my ($value, $noerr) = @_;
1354
1355     return $value if parse_hotplug_features($value);
1356
1357     return if $noerr;
1358
1359     die "unable to parse hotplug option\n";
1360 }
1361
1362 sub scsi_inquiry {
1363     my($fh, $noerr) = @_;
1364
1365     my $SG_IO = 0x2285;
1366     my $SG_GET_VERSION_NUM = 0x2282;
1367
1368     my $versionbuf = "\x00" x 8;
1369     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1370     if (!$ret) {
1371         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1372         return;
1373     }
1374     my $version = unpack("I", $versionbuf);
1375     if ($version < 30000) {
1376         die "scsi generic interface too old\n"  if !$noerr;
1377         return;
1378     }
1379
1380     my $buf = "\x00" x 36;
1381     my $sensebuf = "\x00" x 8;
1382     my $cmd = pack("C x3 C x1", 0x12, 36);
1383
1384     # see /usr/include/scsi/sg.h
1385     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1386
1387     my $packet = pack(
1388         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1389     );
1390
1391     $ret = ioctl($fh, $SG_IO, $packet);
1392     if (!$ret) {
1393         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1394         return;
1395     }
1396
1397     my @res = unpack($sg_io_hdr_t, $packet);
1398     if ($res[17] || $res[18]) {
1399         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1400         return;
1401     }
1402
1403     my $res = {};
1404     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1405
1406     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1407     $res->{type} &= 0x1F;
1408
1409     return $res;
1410 }
1411
1412 sub path_is_scsi {
1413     my ($path) = @_;
1414
1415     my $fh = IO::File->new("+<$path") || return;
1416     my $res = scsi_inquiry($fh, 1);
1417     close($fh);
1418
1419     return $res;
1420 }
1421
1422 sub print_tabletdevice_full {
1423     my ($conf, $arch) = @_;
1424
1425     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1426
1427     # we use uhci for old VMs because tablet driver was buggy in older qemu
1428     my $usbbus;
1429     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1430         $usbbus = 'ehci';
1431     } else {
1432         $usbbus = 'uhci';
1433     }
1434
1435     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1436 }
1437
1438 sub print_keyboarddevice_full {
1439     my ($conf, $arch) = @_;
1440
1441     return if $arch ne 'aarch64';
1442
1443     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1444 }
1445
1446 my sub get_drive_id {
1447     my ($drive) = @_;
1448     return "$drive->{interface}$drive->{index}";
1449 }
1450
1451 sub print_drivedevice_full {
1452     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1453
1454     my $device = '';
1455     my $maxdev = 0;
1456
1457     my $drive_id = get_drive_id($drive);
1458     if ($drive->{interface} eq 'virtio') {
1459         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1460         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1461         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1462     } elsif ($drive->{interface} eq 'scsi') {
1463
1464         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1465         my $unit = $drive->{index} % $maxdev;
1466         my $devicetype = 'hd';
1467         my $path = '';
1468         if (drive_is_cdrom($drive)) {
1469             $devicetype = 'cd';
1470         } else {
1471             if ($drive->{file} =~ m|^/|) {
1472                 $path = $drive->{file};
1473                 if (my $info = path_is_scsi($path)) {
1474                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1475                         $devicetype = 'block';
1476                     } elsif ($info->{type} == 1) { # tape
1477                         $devicetype = 'generic';
1478                     }
1479                 }
1480             } else {
1481                  $path = PVE::Storage::path($storecfg, $drive->{file});
1482             }
1483
1484             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1485             my $version = extract_version($machine_type, kvm_user_version());
1486             if ($path =~ m/^iscsi\:\/\// &&
1487                !min_version($version, 4, 1)) {
1488                 $devicetype = 'generic';
1489             }
1490         }
1491
1492         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1493             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1494         } else {
1495             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1496                 .",lun=$drive->{index}";
1497         }
1498         $device .= ",drive=drive-$drive_id,id=$drive_id";
1499
1500         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1501             $device .= ",rotation_rate=1";
1502         }
1503         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1504
1505     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1506         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1507         my $controller = int($drive->{index} / $maxdev);
1508         my $unit = $drive->{index} % $maxdev;
1509         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1510
1511         $device = "ide-$devicetype";
1512         if ($drive->{interface} eq 'ide') {
1513             $device .= ",bus=ide.$controller,unit=$unit";
1514         } else {
1515             $device .= ",bus=ahci$controller.$unit";
1516         }
1517         $device .= ",drive=drive-$drive_id,id=$drive_id";
1518
1519         if ($devicetype eq 'hd') {
1520             if (my $model = $drive->{model}) {
1521                 $model = URI::Escape::uri_unescape($model);
1522                 $device .= ",model=$model";
1523             }
1524             if ($drive->{ssd}) {
1525                 $device .= ",rotation_rate=1";
1526             }
1527         }
1528         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1529     } elsif ($drive->{interface} eq 'usb') {
1530         die "implement me";
1531         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1532     } else {
1533         die "unsupported interface type";
1534     }
1535
1536     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1537
1538     if (my $serial = $drive->{serial}) {
1539         $serial = URI::Escape::uri_unescape($serial);
1540         $device .= ",serial=$serial";
1541     }
1542
1543
1544     return $device;
1545 }
1546
1547 sub get_initiator_name {
1548     my $initiator;
1549
1550     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1551     while (defined(my $line = <$fh>)) {
1552         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1553         $initiator = $1;
1554         last;
1555     }
1556     $fh->close();
1557
1558     return $initiator;
1559 }
1560
1561 sub print_drive_commandline_full {
1562     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1563
1564     my $path;
1565     my $volid = $drive->{file};
1566     my $format = $drive->{format};
1567     my $drive_id = get_drive_id($drive);
1568
1569     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1570     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1571
1572     if (drive_is_cdrom($drive)) {
1573         $path = get_iso_path($storecfg, $vmid, $volid);
1574         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1575     } else {
1576         if ($storeid) {
1577             $path = PVE::Storage::path($storecfg, $volid);
1578             $format //= qemu_img_format($scfg, $volname);
1579         } else {
1580             $path = $volid;
1581             $format //= "raw";
1582         }
1583    }
1584
1585    my $is_rbd = $path =~ m/^rbd:/;
1586
1587     my $opts = '';
1588     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1589     foreach my $o (@qemu_drive_options) {
1590         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1591     }
1592
1593     # snapshot only accepts on|off
1594     if (defined($drive->{snapshot})) {
1595         my $v = $drive->{snapshot} ? 'on' : 'off';
1596         $opts .= ",snapshot=$v";
1597     }
1598
1599     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1600         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1601     }
1602
1603     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1604         my ($dir, $qmpname) = @$type;
1605         if (my $v = $drive->{"mbps$dir"}) {
1606             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1607         }
1608         if (my $v = $drive->{"mbps${dir}_max"}) {
1609             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1610         }
1611         if (my $v = $drive->{"bps${dir}_max_length"}) {
1612             $opts .= ",throttling.bps$qmpname-max-length=$v";
1613         }
1614         if (my $v = $drive->{"iops${dir}"}) {
1615             $opts .= ",throttling.iops$qmpname=$v";
1616         }
1617         if (my $v = $drive->{"iops${dir}_max"}) {
1618             $opts .= ",throttling.iops$qmpname-max=$v";
1619         }
1620         if (my $v = $drive->{"iops${dir}_max_length"}) {
1621             $opts .= ",throttling.iops$qmpname-max-length=$v";
1622         }
1623     }
1624
1625     if ($pbs_name) {
1626         $format = "rbd" if $is_rbd;
1627         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1628             if !$format;
1629         $opts .= ",format=alloc-track,file.driver=$format";
1630     } elsif ($format) {
1631         $opts .= ",format=$format";
1632     }
1633
1634     my $cache_direct = 0;
1635
1636     if (my $cache = $drive->{cache}) {
1637         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1638     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1639         $opts .= ",cache=none";
1640         $cache_direct = 1;
1641     }
1642
1643     # io_uring with cache mode writeback or writethrough on krbd will hang...
1644     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1645
1646     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1647     # sometimes, just plain disable...
1648     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1649
1650     if (!$drive->{aio}) {
1651         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1652             # io_uring supports all cache modes
1653             $opts .= ",aio=io_uring";
1654         } else {
1655             # aio native works only with O_DIRECT
1656             if($cache_direct) {
1657                 $opts .= ",aio=native";
1658             } else {
1659                 $opts .= ",aio=threads";
1660             }
1661         }
1662     }
1663
1664     if (!drive_is_cdrom($drive)) {
1665         my $detectzeroes;
1666         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1667             $detectzeroes = 'off';
1668         } elsif ($drive->{discard}) {
1669             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1670         } else {
1671             # This used to be our default with discard not being specified:
1672             $detectzeroes = 'on';
1673         }
1674
1675         # note: 'detect-zeroes' works per blockdev and we want it to persist
1676         # after the alloc-track is removed, so put it on 'file' directly
1677         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1678         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1679     }
1680
1681     if ($pbs_name) {
1682         $opts .= ",backing=$pbs_name";
1683         $opts .= ",auto-remove=on";
1684     }
1685
1686     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1687     my $file_param = "file";
1688     if ($pbs_name) {
1689         # non-rbd drivers require the underlying file to be a seperate block
1690         # node, so add a second .file indirection
1691         $file_param .= ".file" if !$is_rbd;
1692         $file_param .= ".filename";
1693     }
1694     my $pathinfo = $path ? "$file_param=$path," : '';
1695
1696     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1697 }
1698
1699 sub print_pbs_blockdev {
1700     my ($pbs_conf, $pbs_name) = @_;
1701     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1702     $blockdev .= ",repository=$pbs_conf->{repository}";
1703     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1704     $blockdev .= ",archive=$pbs_conf->{archive}";
1705     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1706     return $blockdev;
1707 }
1708
1709 sub print_netdevice_full {
1710     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1711
1712     my $device = $net->{model};
1713     if ($net->{model} eq 'virtio') {
1714          $device = 'virtio-net-pci';
1715      };
1716
1717     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1718     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1719     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1720         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1721         # and out of each queue plus one config interrupt and control vector queue
1722         my $vectors = $net->{queues} * 2 + 2;
1723         $tmpstr .= ",vectors=$vectors,mq=on";
1724     }
1725     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1726
1727     if (my $mtu = $net->{mtu}) {
1728         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1729             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1730             if ($mtu == 1) {
1731                  $mtu = $bridge_mtu;
1732             } elsif ($mtu < 576) {
1733                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1734             } elsif ($mtu > $bridge_mtu) {
1735                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1736             }
1737             $tmpstr .= ",host_mtu=$mtu";
1738         } else {
1739             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1740         }
1741     }
1742
1743     if ($use_old_bios_files) {
1744         my $romfile;
1745         if ($device eq 'virtio-net-pci') {
1746             $romfile = 'pxe-virtio.rom';
1747         } elsif ($device eq 'e1000') {
1748             $romfile = 'pxe-e1000.rom';
1749         } elsif ($device eq 'e1000e') {
1750             $romfile = 'pxe-e1000e.rom';
1751         } elsif ($device eq 'ne2k') {
1752             $romfile = 'pxe-ne2k_pci.rom';
1753         } elsif ($device eq 'pcnet') {
1754             $romfile = 'pxe-pcnet.rom';
1755         } elsif ($device eq 'rtl8139') {
1756             $romfile = 'pxe-rtl8139.rom';
1757         }
1758         $tmpstr .= ",romfile=$romfile" if $romfile;
1759     }
1760
1761     return $tmpstr;
1762 }
1763
1764 sub print_netdev_full {
1765     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1766
1767     my $i = '';
1768     if ($netid =~ m/^net(\d+)$/) {
1769         $i = int($1);
1770     }
1771
1772     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1773
1774     my $ifname = "tap${vmid}i$i";
1775
1776     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1777     die "interface name '$ifname' is too long (max 15 character)\n"
1778         if length($ifname) >= 16;
1779
1780     my $vhostparam = '';
1781     if (is_native($arch)) {
1782         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1783     }
1784
1785     my $vmname = $conf->{name} || "vm$vmid";
1786
1787     my $netdev = "";
1788     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1789
1790     if ($net->{bridge}) {
1791         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1792             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1793     } else {
1794         $netdev = "type=user,id=$netid,hostname=$vmname";
1795     }
1796
1797     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1798
1799     return $netdev;
1800 }
1801
1802 my $vga_map = {
1803     'cirrus' => 'cirrus-vga',
1804     'std' => 'VGA',
1805     'vmware' => 'vmware-svga',
1806     'virtio' => 'virtio-vga',
1807 };
1808
1809 sub print_vga_device {
1810     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1811
1812     my $type = $vga_map->{$vga->{type}};
1813     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1814         $type = 'virtio-gpu';
1815     }
1816     my $vgamem_mb = $vga->{memory};
1817
1818     my $max_outputs = '';
1819     if ($qxlnum) {
1820         $type = $id ? 'qxl' : 'qxl-vga';
1821
1822         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1823             # set max outputs so linux can have up to 4 qxl displays with one device
1824             if (min_version($machine_version, 4, 1)) {
1825                 $max_outputs = ",max_outputs=4";
1826             }
1827         }
1828     }
1829
1830     die "no devicetype for $vga->{type}\n" if !$type;
1831
1832     my $memory = "";
1833     if ($vgamem_mb) {
1834         if ($vga->{type} eq 'virtio') {
1835             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1836             $memory = ",max_hostmem=$bytes";
1837         } elsif ($qxlnum) {
1838             # from https://www.spice-space.org/multiple-monitors.html
1839             $memory = ",vgamem_mb=$vga->{memory}";
1840             my $ram = $vgamem_mb * 4;
1841             my $vram = $vgamem_mb * 2;
1842             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1843         } else {
1844             $memory = ",vgamem_mb=$vga->{memory}";
1845         }
1846     } elsif ($qxlnum && $id) {
1847         $memory = ",ram_size=67108864,vram_size=33554432";
1848     }
1849
1850     my $edidoff = "";
1851     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1852         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1853     }
1854
1855     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1856     my $vgaid = "vga" . ($id // '');
1857     my $pciaddr;
1858     if ($q35 && $vgaid eq 'vga') {
1859         # the first display uses pcie.0 bus on q35 machines
1860         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1861     } else {
1862         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1863     }
1864
1865     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1866 }
1867
1868 sub parse_number_sets {
1869     my ($set) = @_;
1870     my $res = [];
1871     foreach my $part (split(/;/, $set)) {
1872         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1873             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1874             push @$res, [ $1, $2 ];
1875         } else {
1876             die "invalid range: $part\n";
1877         }
1878     }
1879     return $res;
1880 }
1881
1882 sub parse_numa {
1883     my ($data) = @_;
1884
1885     my $res = parse_property_string($numa_fmt, $data);
1886     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1887     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1888     return $res;
1889 }
1890
1891 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1892 sub parse_net {
1893     my ($data) = @_;
1894
1895     my $res = eval { parse_property_string($net_fmt, $data) };
1896     if ($@) {
1897         warn $@;
1898         return;
1899     }
1900     if (!defined($res->{macaddr})) {
1901         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1902         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1903     }
1904     return $res;
1905 }
1906
1907 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1908 sub parse_ipconfig {
1909     my ($data) = @_;
1910
1911     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1912     if ($@) {
1913         warn $@;
1914         return;
1915     }
1916
1917     if ($res->{gw} && !$res->{ip}) {
1918         warn 'gateway specified without specifying an IP address';
1919         return;
1920     }
1921     if ($res->{gw6} && !$res->{ip6}) {
1922         warn 'IPv6 gateway specified without specifying an IPv6 address';
1923         return;
1924     }
1925     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1926         warn 'gateway specified together with DHCP';
1927         return;
1928     }
1929     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1930         # gw6 + auto/dhcp
1931         warn "IPv6 gateway specified together with $res->{ip6} address";
1932         return;
1933     }
1934
1935     if (!$res->{ip} && !$res->{ip6}) {
1936         return { ip => 'dhcp', ip6 => 'dhcp' };
1937     }
1938
1939     return $res;
1940 }
1941
1942 sub print_net {
1943     my $net = shift;
1944
1945     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1946 }
1947
1948 sub add_random_macs {
1949     my ($settings) = @_;
1950
1951     foreach my $opt (keys %$settings) {
1952         next if $opt !~ m/^net(\d+)$/;
1953         my $net = parse_net($settings->{$opt});
1954         next if !$net;
1955         $settings->{$opt} = print_net($net);
1956     }
1957 }
1958
1959 sub vm_is_volid_owner {
1960     my ($storecfg, $vmid, $volid) = @_;
1961
1962     if ($volid !~  m|^/|) {
1963         my ($path, $owner);
1964         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1965         if ($owner && ($owner == $vmid)) {
1966             return 1;
1967         }
1968     }
1969
1970     return;
1971 }
1972
1973 sub vmconfig_register_unused_drive {
1974     my ($storecfg, $vmid, $conf, $drive) = @_;
1975
1976     if (drive_is_cloudinit($drive)) {
1977         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1978         warn $@ if $@;
1979     } elsif (!drive_is_cdrom($drive)) {
1980         my $volid = $drive->{file};
1981         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1982             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1983         }
1984     }
1985 }
1986
1987 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1988 my $smbios1_fmt = {
1989     uuid => {
1990         type => 'string',
1991         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1992         format_description => 'UUID',
1993         description => "Set SMBIOS1 UUID.",
1994         optional => 1,
1995     },
1996     version => {
1997         type => 'string',
1998         pattern => '[A-Za-z0-9+\/]+={0,2}',
1999         format_description => 'Base64 encoded string',
2000         description => "Set SMBIOS1 version.",
2001         optional => 1,
2002     },
2003     serial => {
2004         type => 'string',
2005         pattern => '[A-Za-z0-9+\/]+={0,2}',
2006         format_description => 'Base64 encoded string',
2007         description => "Set SMBIOS1 serial number.",
2008         optional => 1,
2009     },
2010     manufacturer => {
2011         type => 'string',
2012         pattern => '[A-Za-z0-9+\/]+={0,2}',
2013         format_description => 'Base64 encoded string',
2014         description => "Set SMBIOS1 manufacturer.",
2015         optional => 1,
2016     },
2017     product => {
2018         type => 'string',
2019         pattern => '[A-Za-z0-9+\/]+={0,2}',
2020         format_description => 'Base64 encoded string',
2021         description => "Set SMBIOS1 product ID.",
2022         optional => 1,
2023     },
2024     sku => {
2025         type => 'string',
2026         pattern => '[A-Za-z0-9+\/]+={0,2}',
2027         format_description => 'Base64 encoded string',
2028         description => "Set SMBIOS1 SKU string.",
2029         optional => 1,
2030     },
2031     family => {
2032         type => 'string',
2033         pattern => '[A-Za-z0-9+\/]+={0,2}',
2034         format_description => 'Base64 encoded string',
2035         description => "Set SMBIOS1 family string.",
2036         optional => 1,
2037     },
2038     base64 => {
2039         type => 'boolean',
2040         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2041         optional => 1,
2042     },
2043 };
2044
2045 sub parse_smbios1 {
2046     my ($data) = @_;
2047
2048     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2049     warn $@ if $@;
2050     return $res;
2051 }
2052
2053 sub print_smbios1 {
2054     my ($smbios1) = @_;
2055     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2056 }
2057
2058 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2059
2060 sub parse_watchdog {
2061     my ($value) = @_;
2062
2063     return if !$value;
2064
2065     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2066     warn $@ if $@;
2067     return $res;
2068 }
2069
2070 sub parse_guest_agent {
2071     my ($conf) = @_;
2072
2073     return {} if !defined($conf->{agent});
2074
2075     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2076     warn $@ if $@;
2077
2078     # if the agent is disabled ignore the other potentially set properties
2079     return {} if !$res->{enabled};
2080     return $res;
2081 }
2082
2083 sub get_qga_key {
2084     my ($conf, $key) = @_;
2085     return undef if !defined($conf->{agent});
2086
2087     my $agent = parse_guest_agent($conf);
2088     return $agent->{$key};
2089 }
2090
2091 sub parse_vga {
2092     my ($value) = @_;
2093
2094     return {} if !$value;
2095     my $res = eval { parse_property_string($vga_fmt, $value) };
2096     warn $@ if $@;
2097     return $res;
2098 }
2099
2100 sub parse_rng {
2101     my ($value) = @_;
2102
2103     return if !$value;
2104
2105     my $res = eval { parse_property_string($rng_fmt, $value) };
2106     warn $@ if $@;
2107     return $res;
2108 }
2109
2110 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2111 sub verify_usb_device {
2112     my ($value, $noerr) = @_;
2113
2114     return $value if parse_usb_device($value);
2115
2116     return if $noerr;
2117
2118     die "unable to parse usb device\n";
2119 }
2120
2121 # add JSON properties for create and set function
2122 sub json_config_properties {
2123     my $prop = shift;
2124
2125     my $skip_json_config_opts = {
2126         parent => 1,
2127         snaptime => 1,
2128         vmstate => 1,
2129         runningmachine => 1,
2130         runningcpu => 1,
2131     };
2132
2133     foreach my $opt (keys %$confdesc) {
2134         next if $skip_json_config_opts->{$opt};
2135         $prop->{$opt} = $confdesc->{$opt};
2136     }
2137
2138     return $prop;
2139 }
2140
2141 # return copy of $confdesc_cloudinit to generate documentation
2142 sub cloudinit_config_properties {
2143
2144     return dclone($confdesc_cloudinit);
2145 }
2146
2147 sub check_type {
2148     my ($key, $value) = @_;
2149
2150     die "unknown setting '$key'\n" if !$confdesc->{$key};
2151
2152     my $type = $confdesc->{$key}->{type};
2153
2154     if (!defined($value)) {
2155         die "got undefined value\n";
2156     }
2157
2158     if ($value =~ m/[\n\r]/) {
2159         die "property contains a line feed\n";
2160     }
2161
2162     if ($type eq 'boolean') {
2163         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2164         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2165         die "type check ('boolean') failed - got '$value'\n";
2166     } elsif ($type eq 'integer') {
2167         return int($1) if $value =~ m/^(\d+)$/;
2168         die "type check ('integer') failed - got '$value'\n";
2169     } elsif ($type eq 'number') {
2170         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2171         die "type check ('number') failed - got '$value'\n";
2172     } elsif ($type eq 'string') {
2173         if (my $fmt = $confdesc->{$key}->{format}) {
2174             PVE::JSONSchema::check_format($fmt, $value);
2175             return $value;
2176         }
2177         $value =~ s/^\"(.*)\"$/$1/;
2178         return $value;
2179     } else {
2180         die "internal error"
2181     }
2182 }
2183
2184 sub destroy_vm {
2185     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2186
2187     my $conf = PVE::QemuConfig->load_config($vmid);
2188
2189     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2190
2191     if ($conf->{template}) {
2192         # check if any base image is still used by a linked clone
2193         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2194                 my ($ds, $drive) = @_;
2195                 return if drive_is_cdrom($drive);
2196
2197                 my $volid = $drive->{file};
2198                 return if !$volid || $volid =~ m|^/|;
2199
2200                 die "base volume '$volid' is still in use by linked cloned\n"
2201                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2202
2203         });
2204     }
2205
2206     my $volids = {};
2207     my $remove_owned_drive = sub {
2208         my ($ds, $drive) = @_;
2209         return if drive_is_cdrom($drive, 1);
2210
2211         my $volid = $drive->{file};
2212         return if !$volid || $volid =~ m|^/|;
2213         return if $volids->{$volid};
2214
2215         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2216         return if !$path || !$owner || ($owner != $vmid);
2217
2218         $volids->{$volid} = 1;
2219         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2220         warn "Could not remove disk '$volid', check manually: $@" if $@;
2221     };
2222
2223     # only remove disks owned by this VM (referenced in the config)
2224     my $include_opts = {
2225         include_unused => 1,
2226         extra_keys => ['vmstate'],
2227     };
2228     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2229
2230     for my $snap (values %{$conf->{snapshots}}) {
2231         next if !defined($snap->{vmstate});
2232         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2233         next if !defined($drive);
2234         $remove_owned_drive->('vmstate', $drive);
2235     }
2236
2237     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2238
2239     if ($purge_unreferenced) { # also remove unreferenced disk
2240         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2241         PVE::Storage::foreach_volid($vmdisks, sub {
2242             my ($volid, $sid, $volname, $d) = @_;
2243             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2244             warn $@ if $@;
2245         });
2246     }
2247
2248     if (defined $replacement_conf) {
2249         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2250     } else {
2251         PVE::QemuConfig->destroy_config($vmid);
2252     }
2253 }
2254
2255 sub parse_vm_config {
2256     my ($filename, $raw) = @_;
2257
2258     return if !defined($raw);
2259
2260     my $res = {
2261         digest => Digest::SHA::sha1_hex($raw),
2262         snapshots => {},
2263         pending => {},
2264     };
2265
2266     $filename =~ m|/qemu-server/(\d+)\.conf$|
2267         || die "got strange filename '$filename'";
2268
2269     my $vmid = $1;
2270
2271     my $conf = $res;
2272     my $descr;
2273     my $section = '';
2274
2275     my @lines = split(/\n/, $raw);
2276     foreach my $line (@lines) {
2277         next if $line =~ m/^\s*$/;
2278
2279         if ($line =~ m/^\[PENDING\]\s*$/i) {
2280             $section = 'pending';
2281             if (defined($descr)) {
2282                 $descr =~ s/\s+$//;
2283                 $conf->{description} = $descr;
2284             }
2285             $descr = undef;
2286             $conf = $res->{$section} = {};
2287             next;
2288
2289         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2290             $section = $1;
2291             if (defined($descr)) {
2292                 $descr =~ s/\s+$//;
2293                 $conf->{description} = $descr;
2294             }
2295             $descr = undef;
2296             $conf = $res->{snapshots}->{$section} = {};
2297             next;
2298         }
2299
2300         if ($line =~ m/^\#(.*)\s*$/) {
2301             $descr = '' if !defined($descr);
2302             $descr .= PVE::Tools::decode_text($1) . "\n";
2303             next;
2304         }
2305
2306         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2307             $descr = '' if !defined($descr);
2308             $descr .= PVE::Tools::decode_text($2);
2309         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2310             $conf->{snapstate} = $1;
2311         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2312             my $key = $1;
2313             my $value = $2;
2314             $conf->{$key} = $value;
2315         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2316             my $value = $1;
2317             if ($section eq 'pending') {
2318                 $conf->{delete} = $value; # we parse this later
2319             } else {
2320                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2321             }
2322         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2323             my $key = $1;
2324             my $value = $2;
2325             eval { $value = check_type($key, $value); };
2326             if ($@) {
2327                 warn "vm $vmid - unable to parse value of '$key' - $@";
2328             } else {
2329                 $key = 'ide2' if $key eq 'cdrom';
2330                 my $fmt = $confdesc->{$key}->{format};
2331                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2332                     my $v = parse_drive($key, $value);
2333                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2334                         $v->{file} = $volid;
2335                         $value = print_drive($v);
2336                     } else {
2337                         warn "vm $vmid - unable to parse value of '$key'\n";
2338                         next;
2339                     }
2340                 }
2341
2342                 $conf->{$key} = $value;
2343             }
2344         } else {
2345             warn "vm $vmid - unable to parse config: $line\n";
2346         }
2347     }
2348
2349     if (defined($descr)) {
2350         $descr =~ s/\s+$//;
2351         $conf->{description} = $descr;
2352     }
2353     delete $res->{snapstate}; # just to be sure
2354
2355     return $res;
2356 }
2357
2358 sub write_vm_config {
2359     my ($filename, $conf) = @_;
2360
2361     delete $conf->{snapstate}; # just to be sure
2362
2363     if ($conf->{cdrom}) {
2364         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2365         $conf->{ide2} = $conf->{cdrom};
2366         delete $conf->{cdrom};
2367     }
2368
2369     # we do not use 'smp' any longer
2370     if ($conf->{sockets}) {
2371         delete $conf->{smp};
2372     } elsif ($conf->{smp}) {
2373         $conf->{sockets} = $conf->{smp};
2374         delete $conf->{cores};
2375         delete $conf->{smp};
2376     }
2377
2378     my $used_volids = {};
2379
2380     my $cleanup_config = sub {
2381         my ($cref, $pending, $snapname) = @_;
2382
2383         foreach my $key (keys %$cref) {
2384             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2385                 $key eq 'snapstate' || $key eq 'pending';
2386             my $value = $cref->{$key};
2387             if ($key eq 'delete') {
2388                 die "propertry 'delete' is only allowed in [PENDING]\n"
2389                     if !$pending;
2390                 # fixme: check syntax?
2391                 next;
2392             }
2393             eval { $value = check_type($key, $value); };
2394             die "unable to parse value of '$key' - $@" if $@;
2395
2396             $cref->{$key} = $value;
2397
2398             if (!$snapname && is_valid_drivename($key)) {
2399                 my $drive = parse_drive($key, $value);
2400                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2401             }
2402         }
2403     };
2404
2405     &$cleanup_config($conf);
2406
2407     &$cleanup_config($conf->{pending}, 1);
2408
2409     foreach my $snapname (keys %{$conf->{snapshots}}) {
2410         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2411         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2412     }
2413
2414     # remove 'unusedX' settings if we re-add a volume
2415     foreach my $key (keys %$conf) {
2416         my $value = $conf->{$key};
2417         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2418             delete $conf->{$key};
2419         }
2420     }
2421
2422     my $generate_raw_config = sub {
2423         my ($conf, $pending) = @_;
2424
2425         my $raw = '';
2426
2427         # add description as comment to top of file
2428         if (defined(my $descr = $conf->{description})) {
2429             if ($descr) {
2430                 foreach my $cl (split(/\n/, $descr)) {
2431                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2432                 }
2433             } else {
2434                 $raw .= "#\n" if $pending;
2435             }
2436         }
2437
2438         foreach my $key (sort keys %$conf) {
2439             next if $key =~ /^(digest|description|pending|snapshots)$/;
2440             $raw .= "$key: $conf->{$key}\n";
2441         }
2442         return $raw;
2443     };
2444
2445     my $raw = &$generate_raw_config($conf);
2446
2447     if (scalar(keys %{$conf->{pending}})){
2448         $raw .= "\n[PENDING]\n";
2449         $raw .= &$generate_raw_config($conf->{pending}, 1);
2450     }
2451
2452     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2453         $raw .= "\n[$snapname]\n";
2454         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2455     }
2456
2457     return $raw;
2458 }
2459
2460 sub load_defaults {
2461
2462     my $res = {};
2463
2464     # we use static defaults from our JSON schema configuration
2465     foreach my $key (keys %$confdesc) {
2466         if (defined(my $default = $confdesc->{$key}->{default})) {
2467             $res->{$key} = $default;
2468         }
2469     }
2470
2471     return $res;
2472 }
2473
2474 sub config_list {
2475     my $vmlist = PVE::Cluster::get_vmlist();
2476     my $res = {};
2477     return $res if !$vmlist || !$vmlist->{ids};
2478     my $ids = $vmlist->{ids};
2479     my $nodename = nodename();
2480
2481     foreach my $vmid (keys %$ids) {
2482         my $d = $ids->{$vmid};
2483         next if !$d->{node} || $d->{node} ne $nodename;
2484         next if !$d->{type} || $d->{type} ne 'qemu';
2485         $res->{$vmid}->{exists} = 1;
2486     }
2487     return $res;
2488 }
2489
2490 # test if VM uses local resources (to prevent migration)
2491 sub check_local_resources {
2492     my ($conf, $noerr) = @_;
2493
2494     my @loc_res = ();
2495
2496     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2497     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2498
2499     push @loc_res, "ivshmem" if $conf->{ivshmem};
2500
2501     foreach my $k (keys %$conf) {
2502         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2503         # sockets are safe: they will recreated be on the target side post-migrate
2504         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2505         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2506     }
2507
2508     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2509
2510     return \@loc_res;
2511 }
2512
2513 # check if used storages are available on all nodes (use by migrate)
2514 sub check_storage_availability {
2515     my ($storecfg, $conf, $node) = @_;
2516
2517     PVE::QemuConfig->foreach_volume($conf, sub {
2518         my ($ds, $drive) = @_;
2519
2520         my $volid = $drive->{file};
2521         return if !$volid;
2522
2523         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2524         return if !$sid;
2525
2526         # check if storage is available on both nodes
2527         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2528         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2529
2530         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2531
2532         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2533             if !$scfg->{content}->{$vtype};
2534    });
2535 }
2536
2537 # list nodes where all VM images are available (used by has_feature API)
2538 sub shared_nodes {
2539     my ($conf, $storecfg) = @_;
2540
2541     my $nodelist = PVE::Cluster::get_nodelist();
2542     my $nodehash = { map { $_ => 1 } @$nodelist };
2543     my $nodename = nodename();
2544
2545     PVE::QemuConfig->foreach_volume($conf, sub {
2546         my ($ds, $drive) = @_;
2547
2548         my $volid = $drive->{file};
2549         return if !$volid;
2550
2551         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2552         if ($storeid) {
2553             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2554             if ($scfg->{disable}) {
2555                 $nodehash = {};
2556             } elsif (my $avail = $scfg->{nodes}) {
2557                 foreach my $node (keys %$nodehash) {
2558                     delete $nodehash->{$node} if !$avail->{$node};
2559                 }
2560             } elsif (!$scfg->{shared}) {
2561                 foreach my $node (keys %$nodehash) {
2562                     delete $nodehash->{$node} if $node ne $nodename
2563                 }
2564             }
2565         }
2566     });
2567
2568     return $nodehash
2569 }
2570
2571 sub check_local_storage_availability {
2572     my ($conf, $storecfg) = @_;
2573
2574     my $nodelist = PVE::Cluster::get_nodelist();
2575     my $nodehash = { map { $_ => {} } @$nodelist };
2576
2577     PVE::QemuConfig->foreach_volume($conf, sub {
2578         my ($ds, $drive) = @_;
2579
2580         my $volid = $drive->{file};
2581         return if !$volid;
2582
2583         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2584         if ($storeid) {
2585             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2586
2587             if ($scfg->{disable}) {
2588                 foreach my $node (keys %$nodehash) {
2589                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2590                 }
2591             } elsif (my $avail = $scfg->{nodes}) {
2592                 foreach my $node (keys %$nodehash) {
2593                     if (!$avail->{$node}) {
2594                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2595                     }
2596                 }
2597             }
2598         }
2599     });
2600
2601     foreach my $node (values %$nodehash) {
2602         if (my $unavail = $node->{unavailable_storages}) {
2603             $node->{unavailable_storages} = [ sort keys %$unavail ];
2604         }
2605     }
2606
2607     return $nodehash
2608 }
2609
2610 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2611 sub check_running {
2612     my ($vmid, $nocheck, $node) = @_;
2613
2614     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2615     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2616 }
2617
2618 sub vzlist {
2619
2620     my $vzlist = config_list();
2621
2622     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2623
2624     while (defined(my $de = $fd->read)) {
2625         next if $de !~ m/^(\d+)\.pid$/;
2626         my $vmid = $1;
2627         next if !defined($vzlist->{$vmid});
2628         if (my $pid = check_running($vmid)) {
2629             $vzlist->{$vmid}->{pid} = $pid;
2630         }
2631     }
2632
2633     return $vzlist;
2634 }
2635
2636 our $vmstatus_return_properties = {
2637     vmid => get_standard_option('pve-vmid'),
2638     status => {
2639         description => "Qemu process status.",
2640         type => 'string',
2641         enum => ['stopped', 'running'],
2642     },
2643     maxmem => {
2644         description => "Maximum memory in bytes.",
2645         type => 'integer',
2646         optional => 1,
2647         renderer => 'bytes',
2648     },
2649     maxdisk => {
2650         description => "Root disk size in bytes.",
2651         type => 'integer',
2652         optional => 1,
2653         renderer => 'bytes',
2654     },
2655     name => {
2656         description => "VM name.",
2657         type => 'string',
2658         optional => 1,
2659     },
2660     qmpstatus => {
2661         description => "Qemu QMP agent status.",
2662         type => 'string',
2663         optional => 1,
2664     },
2665     pid => {
2666         description => "PID of running qemu process.",
2667         type => 'integer',
2668         optional => 1,
2669     },
2670     uptime => {
2671         description => "Uptime.",
2672         type => 'integer',
2673         optional => 1,
2674         renderer => 'duration',
2675     },
2676     cpus => {
2677         description => "Maximum usable CPUs.",
2678         type => 'number',
2679         optional => 1,
2680     },
2681     lock => {
2682         description => "The current config lock, if any.",
2683         type => 'string',
2684         optional => 1,
2685     },
2686     tags => {
2687         description  => "The current configured tags, if any",
2688         type => 'string',
2689         optional => 1,
2690     },
2691     'running-machine' => {
2692         description => "The currently running machine type (if running).",
2693         type => 'string',
2694         optional => 1,
2695     },
2696     'running-qemu' => {
2697         description => "The currently running QEMU version (if running).",
2698         type => 'string',
2699         optional => 1,
2700     },
2701 };
2702
2703 my $last_proc_pid_stat;
2704
2705 # get VM status information
2706 # This must be fast and should not block ($full == false)
2707 # We only query KVM using QMP if $full == true (this can be slow)
2708 sub vmstatus {
2709     my ($opt_vmid, $full) = @_;
2710
2711     my $res = {};
2712
2713     my $storecfg = PVE::Storage::config();
2714
2715     my $list = vzlist();
2716     my $defaults = load_defaults();
2717
2718     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2719
2720     my $cpucount = $cpuinfo->{cpus} || 1;
2721
2722     foreach my $vmid (keys %$list) {
2723         next if $opt_vmid && ($vmid ne $opt_vmid);
2724
2725         my $conf = PVE::QemuConfig->load_config($vmid);
2726
2727         my $d = { vmid => int($vmid) };
2728         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2729
2730         # fixme: better status?
2731         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2732
2733         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2734         if (defined($size)) {
2735             $d->{disk} = 0; # no info available
2736             $d->{maxdisk} = $size;
2737         } else {
2738             $d->{disk} = 0;
2739             $d->{maxdisk} = 0;
2740         }
2741
2742         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2743             * ($conf->{cores} || $defaults->{cores});
2744         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2745         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2746
2747         $d->{name} = $conf->{name} || "VM $vmid";
2748         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2749             : $defaults->{memory}*(1024*1024);
2750
2751         if ($conf->{balloon}) {
2752             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2753             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2754                 : $defaults->{shares};
2755         }
2756
2757         $d->{uptime} = 0;
2758         $d->{cpu} = 0;
2759         $d->{mem} = 0;
2760
2761         $d->{netout} = 0;
2762         $d->{netin} = 0;
2763
2764         $d->{diskread} = 0;
2765         $d->{diskwrite} = 0;
2766
2767         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2768
2769         $d->{serial} = 1 if conf_has_serial($conf);
2770         $d->{lock} = $conf->{lock} if $conf->{lock};
2771         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2772
2773         $res->{$vmid} = $d;
2774     }
2775
2776     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2777     foreach my $dev (keys %$netdev) {
2778         next if $dev !~ m/^tap([1-9]\d*)i/;
2779         my $vmid = $1;
2780         my $d = $res->{$vmid};
2781         next if !$d;
2782
2783         $d->{netout} += $netdev->{$dev}->{receive};
2784         $d->{netin} += $netdev->{$dev}->{transmit};
2785
2786         if ($full) {
2787             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2788             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2789         }
2790
2791     }
2792
2793     my $ctime = gettimeofday;
2794
2795     foreach my $vmid (keys %$list) {
2796
2797         my $d = $res->{$vmid};
2798         my $pid = $d->{pid};
2799         next if !$pid;
2800
2801         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2802         next if !$pstat; # not running
2803
2804         my $used = $pstat->{utime} + $pstat->{stime};
2805
2806         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2807
2808         if ($pstat->{vsize}) {
2809             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2810         }
2811
2812         my $old = $last_proc_pid_stat->{$pid};
2813         if (!$old) {
2814             $last_proc_pid_stat->{$pid} = {
2815                 time => $ctime,
2816                 used => $used,
2817                 cpu => 0,
2818             };
2819             next;
2820         }
2821
2822         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2823
2824         if ($dtime > 1000) {
2825             my $dutime = $used -  $old->{used};
2826
2827             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2828             $last_proc_pid_stat->{$pid} = {
2829                 time => $ctime,
2830                 used => $used,
2831                 cpu => $d->{cpu},
2832             };
2833         } else {
2834             $d->{cpu} = $old->{cpu};
2835         }
2836     }
2837
2838     return $res if !$full;
2839
2840     my $qmpclient = PVE::QMPClient->new();
2841
2842     my $ballooncb = sub {
2843         my ($vmid, $resp) = @_;
2844
2845         my $info = $resp->{'return'};
2846         return if !$info->{max_mem};
2847
2848         my $d = $res->{$vmid};
2849
2850         # use memory assigned to VM
2851         $d->{maxmem} = $info->{max_mem};
2852         $d->{balloon} = $info->{actual};
2853
2854         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2855             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2856             $d->{freemem} = $info->{free_mem};
2857         }
2858
2859         $d->{ballooninfo} = $info;
2860     };
2861
2862     my $blockstatscb = sub {
2863         my ($vmid, $resp) = @_;
2864         my $data = $resp->{'return'} || [];
2865         my $totalrdbytes = 0;
2866         my $totalwrbytes = 0;
2867
2868         for my $blockstat (@$data) {
2869             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2870             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2871
2872             $blockstat->{device} =~ s/drive-//;
2873             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2874         }
2875         $res->{$vmid}->{diskread} = $totalrdbytes;
2876         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2877     };
2878
2879     my $machinecb = sub {
2880         my ($vmid, $resp) = @_;
2881         my $data = $resp->{'return'} || [];
2882
2883         $res->{$vmid}->{'running-machine'} =
2884             PVE::QemuServer::Machine::current_from_query_machines($data);
2885     };
2886
2887     my $versioncb = sub {
2888         my ($vmid, $resp) = @_;
2889         my $data = $resp->{'return'} // {};
2890         my $version = 'unknown';
2891
2892         if (my $v = $data->{qemu}) {
2893             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2894         }
2895
2896         $res->{$vmid}->{'running-qemu'} = $version;
2897     };
2898
2899     my $statuscb = sub {
2900         my ($vmid, $resp) = @_;
2901
2902         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2903         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2904         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2905         # this fails if ballon driver is not loaded, so this must be
2906         # the last commnand (following command are aborted if this fails).
2907         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2908
2909         my $status = 'unknown';
2910         if (!defined($status = $resp->{'return'}->{status})) {
2911             warn "unable to get VM status\n";
2912             return;
2913         }
2914
2915         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2916     };
2917
2918     foreach my $vmid (keys %$list) {
2919         next if $opt_vmid && ($vmid ne $opt_vmid);
2920         next if !$res->{$vmid}->{pid}; # not running
2921         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2922     }
2923
2924     $qmpclient->queue_execute(undef, 2);
2925
2926     foreach my $vmid (keys %$list) {
2927         next if $opt_vmid && ($vmid ne $opt_vmid);
2928         next if !$res->{$vmid}->{pid}; #not running
2929
2930         # we can't use the $qmpclient since it might have already aborted on
2931         # 'query-balloon', but this might also fail for older versions...
2932         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2933         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2934     }
2935
2936     foreach my $vmid (keys %$list) {
2937         next if $opt_vmid && ($vmid ne $opt_vmid);
2938         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2939     }
2940
2941     return $res;
2942 }
2943
2944 sub conf_has_serial {
2945     my ($conf) = @_;
2946
2947     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2948         if ($conf->{"serial$i"}) {
2949             return 1;
2950         }
2951     }
2952
2953     return 0;
2954 }
2955
2956 sub conf_has_audio {
2957     my ($conf, $id) = @_;
2958
2959     $id //= 0;
2960     my $audio = $conf->{"audio$id"};
2961     return if !defined($audio);
2962
2963     my $audioproperties = parse_property_string($audio_fmt, $audio);
2964     my $audiodriver = $audioproperties->{driver} // 'spice';
2965
2966     return {
2967         dev => $audioproperties->{device},
2968         dev_id => "audiodev$id",
2969         backend => $audiodriver,
2970         backend_id => "$audiodriver-backend${id}",
2971     };
2972 }
2973
2974 sub audio_devs {
2975     my ($audio, $audiopciaddr, $machine_version) = @_;
2976
2977     my $devs = [];
2978
2979     my $id = $audio->{dev_id};
2980     my $audiodev = "";
2981     if (min_version($machine_version, 4, 2)) {
2982         $audiodev = ",audiodev=$audio->{backend_id}";
2983     }
2984
2985     if ($audio->{dev} eq 'AC97') {
2986         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2987     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2988         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2989         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2990         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2991     } else {
2992         die "unkown audio device '$audio->{dev}', implement me!";
2993     }
2994
2995     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2996
2997     return $devs;
2998 }
2999
3000 sub get_tpm_paths {
3001     my ($vmid) = @_;
3002     return {
3003         socket => "/var/run/qemu-server/$vmid.swtpm",
3004         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3005     };
3006 }
3007
3008 sub add_tpm_device {
3009     my ($vmid, $devices, $conf) = @_;
3010
3011     return if !$conf->{tpmstate0};
3012
3013     my $paths = get_tpm_paths($vmid);
3014
3015     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3016     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3017     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3018 }
3019
3020 sub start_swtpm {
3021     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3022
3023     return if !$tpmdrive;
3024
3025     my $state;
3026     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3027     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3028     if ($storeid) {
3029         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3030     } else {
3031         $state = $tpm->{file};
3032     }
3033
3034     my $paths = get_tpm_paths($vmid);
3035
3036     # during migration, we will get state from remote
3037     #
3038     if (!$migration) {
3039         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3040         my $setup_cmd = [
3041             "swtpm_setup",
3042             "--tpmstate",
3043             "file://$state",
3044             "--createek",
3045             "--create-ek-cert",
3046             "--create-platform-cert",
3047             "--lock-nvram",
3048             "--config",
3049             "/etc/swtpm_setup.conf", # do not use XDG configs
3050             "--runas",
3051             "0", # force creation as root, error if not possible
3052             "--not-overwrite", # ignore existing state, do not modify
3053         ];
3054
3055         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3056         # TPM 2.0 supports ECC crypto, use if possible
3057         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3058
3059         run_command($setup_cmd, outfunc => sub {
3060             print "swtpm_setup: $1\n";
3061         });
3062     }
3063
3064     my $emulator_cmd = [
3065         "swtpm",
3066         "socket",
3067         "--tpmstate",
3068         "backend-uri=file://$state,mode=0600",
3069         "--ctrl",
3070         "type=unixio,path=$paths->{socket},mode=0600",
3071         "--pid",
3072         "file=$paths->{pid}",
3073         "--terminate", # terminate on QEMU disconnect
3074         "--daemon",
3075     ];
3076     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3077     run_command($emulator_cmd, outfunc => sub { print $1; });
3078
3079     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3080     while (! -e $paths->{pid}) {
3081         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3082         usleep(50_000);
3083     }
3084
3085     # return untainted PID of swtpm daemon so it can be killed on error
3086     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3087     return $1;
3088 }
3089
3090 sub vga_conf_has_spice {
3091     my ($vga) = @_;
3092
3093     my $vgaconf = parse_vga($vga);
3094     my $vgatype = $vgaconf->{type};
3095     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3096
3097     return $1 || 1;
3098 }
3099
3100 sub is_native($) {
3101     my ($arch) = @_;
3102     return get_host_arch() eq $arch;
3103 }
3104
3105 sub get_vm_arch {
3106     my ($conf) = @_;
3107     return $conf->{arch} // get_host_arch();
3108 }
3109
3110 my $default_machines = {
3111     x86_64 => 'pc',
3112     aarch64 => 'virt',
3113 };
3114
3115 sub get_installed_machine_version {
3116     my ($kvmversion) = @_;
3117     $kvmversion = kvm_user_version() if !defined($kvmversion);
3118     $kvmversion =~ m/^(\d+\.\d+)/;
3119     return $1;
3120 }
3121
3122 sub windows_get_pinned_machine_version {
3123     my ($machine, $base_version, $kvmversion) = @_;
3124
3125     my $pin_version = $base_version;
3126     if (!defined($base_version) ||
3127         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3128     ) {
3129         $pin_version = get_installed_machine_version($kvmversion);
3130     }
3131     if (!$machine || $machine eq 'pc') {
3132         $machine = "pc-i440fx-$pin_version";
3133     } elsif ($machine eq 'q35') {
3134         $machine = "pc-q35-$pin_version";
3135     } elsif ($machine eq 'virt') {
3136         $machine = "virt-$pin_version";
3137     } else {
3138         warn "unknown machine type '$machine', not touching that!\n";
3139     }
3140
3141     return $machine;
3142 }
3143
3144 sub get_vm_machine {
3145     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3146
3147     my $machine = $forcemachine || $conf->{machine};
3148
3149     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3150         $kvmversion //= kvm_user_version();
3151         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3152         # layout which confuses windows quite a bit and may result in various regressions..
3153         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3154         if (windows_version($conf->{ostype})) {
3155             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3156         }
3157         $arch //= 'x86_64';
3158         $machine ||= $default_machines->{$arch};
3159         if ($add_pve_version) {
3160             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3161             $machine .= "+pve$pvever";
3162         }
3163     }
3164
3165     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3166         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3167         $machine = $1 if $is_pxe;
3168
3169         # for version-pinned machines that do not include a pve-version (e.g.
3170         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3171         $machine .= '+pve0';
3172
3173         $machine .= '.pxe' if $is_pxe;
3174     }
3175
3176     return $machine;
3177 }
3178
3179 sub get_ovmf_files($$$) {
3180     my ($arch, $efidisk, $smm) = @_;
3181
3182     my $types = $OVMF->{$arch}
3183         or die "no OVMF images known for architecture '$arch'\n";
3184
3185     my $type = 'default';
3186     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3187         $type = $smm ? "4m" : "4m-no-smm";
3188         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3189     }
3190
3191     return $types->{$type}->@*;
3192 }
3193
3194 my $Arch2Qemu = {
3195     aarch64 => '/usr/bin/qemu-system-aarch64',
3196     x86_64 => '/usr/bin/qemu-system-x86_64',
3197 };
3198 sub get_command_for_arch($) {
3199     my ($arch) = @_;
3200     return '/usr/bin/kvm' if is_native($arch);
3201
3202     my $cmd = $Arch2Qemu->{$arch}
3203         or die "don't know how to emulate architecture '$arch'\n";
3204     return $cmd;
3205 }
3206
3207 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3208 # to use in a QEMU command line (-cpu element), first array_intersect the result
3209 # of query_supported_ with query_understood_. This is necessary because:
3210 #
3211 # a) query_understood_ returns flags the host cannot use and
3212 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3213 #    flags, but CPU settings - with most of them being flags. Those settings
3214 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3215 #
3216 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3217 # expensive. If you need the value returned from this, you can get it much
3218 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3219 # $accel being 'kvm' or 'tcg'.
3220 #
3221 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3222 # changes, automatically populating pmxcfs.
3223 #
3224 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3225 # since kvm and tcg machines support different flags
3226 #
3227 sub query_supported_cpu_flags {
3228     my ($arch) = @_;
3229
3230     $arch //= get_host_arch();
3231     my $default_machine = $default_machines->{$arch};
3232
3233     my $flags = {};
3234
3235     # FIXME: Once this is merged, the code below should work for ARM as well:
3236     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3237     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3238         $arch eq "aarch64";
3239
3240     my $kvm_supported = defined(kvm_version());
3241     my $qemu_cmd = get_command_for_arch($arch);
3242     my $fakevmid = -1;
3243     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3244
3245     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3246     my $query_supported_run_qemu = sub {
3247         my ($kvm) = @_;
3248
3249         my $flags = {};
3250         my $cmd = [
3251             $qemu_cmd,
3252             '-machine', $default_machine,
3253             '-display', 'none',
3254             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3255             '-mon', 'chardev=qmp,mode=control',
3256             '-pidfile', $pidfile,
3257             '-S', '-daemonize'
3258         ];
3259
3260         if (!$kvm) {
3261             push @$cmd, '-accel', 'tcg';
3262         }
3263
3264         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3265         die "QEMU flag querying VM exited with code " . $rc if $rc;
3266
3267         eval {
3268             my $cmd_result = mon_cmd(
3269                 $fakevmid,
3270                 'query-cpu-model-expansion',
3271                 type => 'full',
3272                 model => { name => 'host' }
3273             );
3274
3275             my $props = $cmd_result->{model}->{props};
3276             foreach my $prop (keys %$props) {
3277                 next if $props->{$prop} ne '1';
3278                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3279                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3280                 # We only keep those with underscores, to match /proc/cpuinfo
3281                 $prop =~ s/\.|-/_/g;
3282                 $flags->{$prop} = 1;
3283             }
3284         };
3285         my $err = $@;
3286
3287         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3288         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3289
3290         die $err if $err;
3291
3292         return [ sort keys %$flags ];
3293     };
3294
3295     # We need to query QEMU twice, since KVM and TCG have different supported flags
3296     PVE::QemuConfig->lock_config($fakevmid, sub {
3297         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3298         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3299
3300         if ($kvm_supported) {
3301             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3302             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3303         }
3304     });
3305
3306     return $flags;
3307 }
3308
3309 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3310 my $understood_cpu_flag_dir = "/usr/share/kvm";
3311 sub query_understood_cpu_flags {
3312     my $arch = get_host_arch();
3313     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3314
3315     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3316         if ! -e $filepath;
3317
3318     my $raw = file_get_contents($filepath);
3319     $raw =~ s/^\s+|\s+$//g;
3320     my @flags = split(/\s+/, $raw);
3321
3322     return \@flags;
3323 }
3324
3325 my sub get_cpuunits {
3326     my ($conf) = @_;
3327     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3328 }
3329 sub config_to_command {
3330     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3331         $pbs_backing) = @_;
3332
3333     my $cmd = [];
3334     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3335     my $devices = [];
3336     my $bridges = {};
3337     my $ostype = $conf->{ostype};
3338     my $winversion = windows_version($ostype);
3339     my $kvm = $conf->{kvm};
3340     my $nodename = nodename();
3341
3342     my $arch = get_vm_arch($conf);
3343     my $kvm_binary = get_command_for_arch($arch);
3344     my $kvmver = kvm_user_version($kvm_binary);
3345
3346     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3347         $kvmver //= "undefined";
3348         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3349     }
3350
3351     my $add_pve_version = min_version($kvmver, 4, 1);
3352
3353     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3354     my $machine_version = extract_version($machine_type, $kvmver);
3355     $kvm //= 1 if is_native($arch);
3356
3357     $machine_version =~ m/(\d+)\.(\d+)/;
3358     my ($machine_major, $machine_minor) = ($1, $2);
3359
3360     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3361         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3362     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3363         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3364             ." please upgrade node '$nodename'\n"
3365     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3366         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3367         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3368             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3369             ." node '$nodename'\n";
3370     }
3371
3372     # if a specific +pve version is required for a feature, use $version_guard
3373     # instead of min_version to allow machines to be run with the minimum
3374     # required version
3375     my $required_pve_version = 0;
3376     my $version_guard = sub {
3377         my ($major, $minor, $pve) = @_;
3378         return 0 if !min_version($machine_version, $major, $minor, $pve);
3379         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3380         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3381         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3382         return 1;
3383     };
3384
3385     if ($kvm && !defined kvm_version()) {
3386         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3387             ." or enable in BIOS.\n";
3388     }
3389
3390     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3391     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3392     my $use_old_bios_files = undef;
3393     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3394
3395     my $cpuunits = get_cpuunits($conf);
3396
3397     push @$cmd, $kvm_binary;
3398
3399     push @$cmd, '-id', $vmid;
3400
3401     my $vmname = $conf->{name} || "vm$vmid";
3402
3403     push @$cmd, '-name', $vmname;
3404
3405     push @$cmd, '-no-shutdown';
3406
3407     my $use_virtio = 0;
3408
3409     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3410     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3411     push @$cmd, '-mon', "chardev=qmp,mode=control";
3412
3413     if (min_version($machine_version, 2, 12)) {
3414         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3415         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3416     }
3417
3418     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3419
3420     push @$cmd, '-daemonize';
3421
3422     if ($conf->{smbios1}) {
3423         my $smbios_conf = parse_smbios1($conf->{smbios1});
3424         if ($smbios_conf->{base64}) {
3425             # Do not pass base64 flag to qemu
3426             delete $smbios_conf->{base64};
3427             my $smbios_string = "";
3428             foreach my $key (keys %$smbios_conf) {
3429                 my $value;
3430                 if ($key eq "uuid") {
3431                     $value = $smbios_conf->{uuid}
3432                 } else {
3433                     $value = decode_base64($smbios_conf->{$key});
3434                 }
3435                 # qemu accepts any binary data, only commas need escaping by double comma
3436                 $value =~ s/,/,,/g;
3437                 $smbios_string .= "," . $key . "=" . $value if $value;
3438             }
3439             push @$cmd, '-smbios', "type=1" . $smbios_string;
3440         } else {
3441             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3442         }
3443     }
3444
3445     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3446         my $d;
3447         if (my $efidisk = $conf->{efidisk0}) {
3448             $d = parse_drive('efidisk0', $efidisk);
3449         }
3450
3451         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3452         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3453
3454         my ($path, $format);
3455         my $read_only_str = '';
3456         if ($d) {
3457             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3458             $format = $d->{format};
3459             if ($storeid) {
3460                 $path = PVE::Storage::path($storecfg, $d->{file});
3461                 if (!defined($format)) {
3462                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3463                     $format = qemu_img_format($scfg, $volname);
3464                 }
3465             } else {
3466                 $path = $d->{file};
3467                 die "efidisk format must be specified\n"
3468                     if !defined($format);
3469             }
3470
3471             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3472         } else {
3473             warn "no efidisk configured! Using temporary efivars disk.\n";
3474             $path = "/tmp/$vmid-ovmf.fd";
3475             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3476             $format = 'raw';
3477         }
3478
3479         my $size_str = "";
3480
3481         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3482             $size_str = ",size=" . (-s $ovmf_vars);
3483         }
3484
3485         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3486         my $cache = "";
3487         if ($path =~ m/^rbd:/) {
3488                 $cache = ',cache=writeback';
3489                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3490         }
3491
3492         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3493         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3494     }
3495
3496     if ($q35) { # tell QEMU to load q35 config early
3497         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3498         if (min_version($machine_version, 4, 0)) {
3499             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3500         } else {
3501             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3502         }
3503     }
3504
3505     if ($conf->{vmgenid}) {
3506         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3507     }
3508
3509     # add usb controllers
3510     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3511         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3512     push @$devices, @usbcontrollers if @usbcontrollers;
3513     my $vga = parse_vga($conf->{vga});
3514
3515     my $qxlnum = vga_conf_has_spice($conf->{vga});
3516     $vga->{type} = 'qxl' if $qxlnum;
3517
3518     if (!$vga->{type}) {
3519         if ($arch eq 'aarch64') {
3520             $vga->{type} = 'virtio';
3521         } elsif (min_version($machine_version, 2, 9)) {
3522             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3523         } else {
3524             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3525         }
3526     }
3527
3528     # enable absolute mouse coordinates (needed by vnc)
3529     my $tablet = $conf->{tablet};
3530     if (!defined($tablet)) {
3531         $tablet = $defaults->{tablet};
3532         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3533         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3534     }
3535
3536     if ($tablet) {
3537         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3538         my $kbd = print_keyboarddevice_full($conf, $arch);
3539         push @$devices, '-device', $kbd if defined($kbd);
3540     }
3541
3542     my $bootorder = device_bootorder($conf);
3543
3544     # host pci device passthrough
3545     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3546         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3547
3548     # usb devices
3549     my $usb_dev_features = {};
3550     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3551
3552     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3553         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3554     push @$devices, @usbdevices if @usbdevices;
3555
3556     # serial devices
3557     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3558         my $path = $conf->{"serial$i"} or next;
3559         if ($path eq 'socket') {
3560             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3561             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3562             # On aarch64, serial0 is the UART device. Qemu only allows
3563             # connecting UART devices via the '-serial' command line, as
3564             # the device has a fixed slot on the hardware...
3565             if ($arch eq 'aarch64' && $i == 0) {
3566                 push @$devices, '-serial', "chardev:serial$i";
3567             } else {
3568                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3569             }
3570         } else {
3571             die "no such serial device\n" if ! -c $path;
3572             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3573             push @$devices, '-device', "isa-serial,chardev=serial$i";
3574         }
3575     }
3576
3577     # parallel devices
3578     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3579         if (my $path = $conf->{"parallel$i"}) {
3580             die "no such parallel device\n" if ! -c $path;
3581             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3582             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3583             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3584         }
3585     }
3586
3587     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3588         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3589         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3590         push @$devices, @$audio_devs;
3591     }
3592
3593     add_tpm_device($vmid, $devices, $conf);
3594
3595     my $sockets = 1;
3596     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3597     $sockets = $conf->{sockets} if  $conf->{sockets};
3598
3599     my $cores = $conf->{cores} || 1;
3600
3601     my $maxcpus = $sockets * $cores;
3602
3603     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3604
3605     my $allowed_vcpus = $cpuinfo->{cpus};
3606
3607     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3608
3609     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3610         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3611         for (my $i = 2; $i <= $vcpus; $i++)  {
3612             my $cpustr = print_cpu_device($conf,$i);
3613             push @$cmd, '-device', $cpustr;
3614         }
3615
3616     } else {
3617
3618         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3619     }
3620     push @$cmd, '-nodefaults';
3621
3622     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3623
3624     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3625
3626     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3627
3628     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3629         push @$devices, '-device', print_vga_device(
3630             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3631         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3632         push @$cmd,  '-vnc', "unix:$socket,password=on";
3633     } else {
3634         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3635         push @$cmd, '-nographic';
3636     }
3637
3638     # time drift fix
3639     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3640     my $useLocaltime = $conf->{localtime};
3641
3642     if ($winversion >= 5) { # windows
3643         $useLocaltime = 1 if !defined($conf->{localtime});
3644
3645         # use time drift fix when acpi is enabled
3646         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3647             $tdf = 1 if !defined($conf->{tdf});
3648         }
3649     }
3650
3651     if ($winversion >= 6) {
3652         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3653         push @$cmd, '-no-hpet';
3654     }
3655
3656     push @$rtcFlags, 'driftfix=slew' if $tdf;
3657
3658     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3659         push @$rtcFlags, "base=$conf->{startdate}";
3660     } elsif ($useLocaltime) {
3661         push @$rtcFlags, 'base=localtime';
3662     }
3663
3664     if ($forcecpu) {
3665         push @$cmd, '-cpu', $forcecpu;
3666     } else {
3667         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3668     }
3669
3670     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3671
3672     push @$cmd, '-S' if $conf->{freeze};
3673
3674     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3675
3676     my $guest_agent = parse_guest_agent($conf);
3677
3678     if ($guest_agent->{enabled}) {
3679         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3680         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3681
3682         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3683             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3684             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3685             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3686         } elsif ($guest_agent->{type} eq 'isa') {
3687             push @$devices, '-device', "isa-serial,chardev=qga0";
3688         }
3689     }
3690
3691     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3692     if ($rng && $version_guard->(4, 1, 2)) {
3693         check_rng_source($rng->{source});
3694
3695         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3696         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3697         my $limiter_str = "";
3698         if ($max_bytes) {
3699             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3700         }
3701
3702         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3703         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3704         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3705     }
3706
3707     my $spice_port;
3708
3709     if ($qxlnum) {
3710         if ($qxlnum > 1) {
3711             if ($winversion){
3712                 for (my $i = 1; $i < $qxlnum; $i++){
3713                     push @$devices, '-device', print_vga_device(
3714                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3715                 }
3716             } else {
3717                 # assume other OS works like Linux
3718                 my ($ram, $vram) = ("134217728", "67108864");
3719                 if ($vga->{memory}) {
3720                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3721                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3722                 }
3723                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3724                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3725             }
3726         }
3727
3728         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3729
3730         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3731         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3732         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3733
3734         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3735         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3736         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3737
3738         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3739         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3740
3741         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3742         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3743         if ($spice_enhancement->{foldersharing}) {
3744             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3745             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3746         }
3747
3748         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3749         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3750             if $spice_enhancement->{videostreaming};
3751
3752         push @$devices, '-spice', "$spice_opts";
3753     }
3754
3755     # enable balloon by default, unless explicitly disabled
3756     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3757         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3758         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3759     }
3760
3761     if ($conf->{watchdog}) {
3762         my $wdopts = parse_watchdog($conf->{watchdog});
3763         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3764         my $watchdog = $wdopts->{model} || 'i6300esb';
3765         push @$devices, '-device', "$watchdog$pciaddr";
3766         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3767     }
3768
3769     my $vollist = [];
3770     my $scsicontroller = {};
3771     my $ahcicontroller = {};
3772     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3773
3774     # Add iscsi initiator name if available
3775     if (my $initiator = get_initiator_name()) {
3776         push @$devices, '-iscsi', "initiator-name=$initiator";
3777     }
3778
3779     PVE::QemuConfig->foreach_volume($conf, sub {
3780         my ($ds, $drive) = @_;
3781
3782         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3783             check_volume_storage_type($storecfg, $drive->{file});
3784             push @$vollist, $drive->{file};
3785         }
3786
3787         # ignore efidisk here, already added in bios/fw handling code above
3788         return if $drive->{interface} eq 'efidisk';
3789         # similar for TPM
3790         return if $drive->{interface} eq 'tpmstate';
3791
3792         $use_virtio = 1 if $ds =~ m/^virtio/;
3793
3794         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3795
3796         if ($drive->{interface} eq 'virtio'){
3797            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3798         }
3799
3800         if ($drive->{interface} eq 'scsi') {
3801
3802             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3803
3804             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3805                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3806
3807             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3808             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3809
3810             my $iothread = '';
3811             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3812                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3813                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3814             } elsif ($drive->{iothread}) {
3815                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3816             }
3817
3818             my $queues = '';
3819             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3820                 $queues = ",num_queues=$drive->{queues}";
3821             }
3822
3823             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3824                 if !$scsicontroller->{$controller};
3825             $scsicontroller->{$controller}=1;
3826         }
3827
3828         if ($drive->{interface} eq 'sata') {
3829             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3830             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3831             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3832                 if !$ahcicontroller->{$controller};
3833             $ahcicontroller->{$controller}=1;
3834         }
3835
3836         my $pbs_conf = $pbs_backing->{$ds};
3837         my $pbs_name = undef;
3838         if ($pbs_conf) {
3839             $pbs_name = "drive-$ds-pbs";
3840             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3841         }
3842
3843         my $drive_cmd = print_drive_commandline_full(
3844             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3845
3846         # extra protection for templates, but SATA and IDE don't support it..
3847         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3848
3849         push @$devices, '-drive',$drive_cmd;
3850         push @$devices, '-device', print_drivedevice_full(
3851             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3852     });
3853
3854     for (my $i = 0; $i < $MAX_NETS; $i++) {
3855         my $netname = "net$i";
3856
3857         next if !$conf->{$netname};
3858         my $d = parse_net($conf->{$netname});
3859         next if !$d;
3860
3861         $use_virtio = 1 if $d->{model} eq 'virtio';
3862
3863         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3864
3865         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3866         push @$devices, '-netdev', $netdevfull;
3867
3868         my $netdevicefull = print_netdevice_full(
3869             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3870
3871         push @$devices, '-device', $netdevicefull;
3872     }
3873
3874     if ($conf->{ivshmem}) {
3875         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3876
3877         my $bus;
3878         if ($q35) {
3879             $bus = print_pcie_addr("ivshmem");
3880         } else {
3881             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3882         }
3883
3884         my $ivshmem_name = $ivshmem->{name} // $vmid;
3885         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3886
3887         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3888         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3889             .",size=$ivshmem->{size}M";
3890     }
3891
3892     # pci.4 is nested in pci.1
3893     $bridges->{1} = 1 if $bridges->{4};
3894
3895     if (!$q35) { # add pci bridges
3896         if (min_version($machine_version, 2, 3)) {
3897            $bridges->{1} = 1;
3898            $bridges->{2} = 1;
3899         }
3900         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3901     }
3902
3903     for my $k (sort {$b cmp $a} keys %$bridges) {
3904         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3905
3906         my $k_name = $k;
3907         if ($k == 2 && $legacy_igd) {
3908             $k_name = "$k-igd";
3909         }
3910         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3911         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3912
3913         if ($q35) { # add after -readconfig pve-q35.cfg
3914             splice @$devices, 2, 0, '-device', $devstr;
3915         } else {
3916             unshift @$devices, '-device', $devstr if $k > 0;
3917         }
3918     }
3919
3920     if (!$kvm) {
3921         push @$machineFlags, 'accel=tcg';
3922     }
3923
3924     my $machine_type_min = $machine_type;
3925     if ($add_pve_version) {
3926         $machine_type_min =~ s/\+pve\d+$//;
3927         $machine_type_min .= "+pve$required_pve_version";
3928     }
3929     push @$machineFlags, "type=${machine_type_min}";
3930
3931     push @$cmd, @$devices;
3932     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3933     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3934     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3935
3936     if (my $vmstate = $conf->{vmstate}) {
3937         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3938         push @$vollist, $vmstate;
3939         push @$cmd, '-loadstate', $statepath;
3940         print "activating and using '$vmstate' as vmstate\n";
3941     }
3942
3943     if (PVE::QemuConfig->is_template($conf)) {
3944         # needed to workaround base volumes being read-only
3945         push @$cmd, '-snapshot';
3946     }
3947
3948     # add custom args
3949     if ($conf->{args}) {
3950         my $aa = PVE::Tools::split_args($conf->{args});
3951         push @$cmd, @$aa;
3952     }
3953
3954     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3955 }
3956
3957 sub check_rng_source {
3958     my ($source) = @_;
3959
3960     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3961     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3962         if ! -e $source;
3963
3964     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3965     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3966         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3967         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3968         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3969             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3970             ." to the host.\n";
3971     }
3972 }
3973
3974 sub spice_port {
3975     my ($vmid) = @_;
3976
3977     my $res = mon_cmd($vmid, 'query-spice');
3978
3979     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3980 }
3981
3982 sub vm_devices_list {
3983     my ($vmid) = @_;
3984
3985     my $res = mon_cmd($vmid, 'query-pci');
3986     my $devices_to_check = [];
3987     my $devices = {};
3988     foreach my $pcibus (@$res) {
3989         push @$devices_to_check, @{$pcibus->{devices}},
3990     }
3991
3992     while (@$devices_to_check) {
3993         my $to_check = [];
3994         for my $d (@$devices_to_check) {
3995             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3996             next if !$d->{'pci_bridge'};
3997
3998             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3999             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4000         }
4001         $devices_to_check = $to_check;
4002     }
4003
4004     my $resblock = mon_cmd($vmid, 'query-block');
4005     foreach my $block (@$resblock) {
4006         if($block->{device} =~ m/^drive-(\S+)/){
4007                 $devices->{$1} = 1;
4008         }
4009     }
4010
4011     my $resmice = mon_cmd($vmid, 'query-mice');
4012     foreach my $mice (@$resmice) {
4013         if ($mice->{name} eq 'QEMU HID Tablet') {
4014             $devices->{tablet} = 1;
4015             last;
4016         }
4017     }
4018
4019     # for usb devices there is no query-usb
4020     # but we can iterate over the entries in
4021     # qom-list path=/machine/peripheral
4022     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4023     foreach my $per (@$resperipheral) {
4024         if ($per->{name} =~ m/^usb\d+$/) {
4025             $devices->{$per->{name}} = 1;
4026         }
4027     }
4028
4029     return $devices;
4030 }
4031
4032 sub vm_deviceplug {
4033     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4034
4035     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4036
4037     my $devices_list = vm_devices_list($vmid);
4038     return 1 if defined($devices_list->{$deviceid});
4039
4040     # add PCI bridge if we need it for the device
4041     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4042
4043     if ($deviceid eq 'tablet') {
4044         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4045     } elsif ($deviceid eq 'keyboard') {
4046         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4047     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4048         die "usb hotplug currently not reliable\n";
4049         # since we can't reliably hot unplug all added usb devices and usb
4050         # passthrough breaks live migration we disable usb hotplugging for now
4051         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4052     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4053         qemu_iothread_add($vmid, $deviceid, $device);
4054
4055         qemu_driveadd($storecfg, $vmid, $device);
4056         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4057
4058         qemu_deviceadd($vmid, $devicefull);
4059         eval { qemu_deviceaddverify($vmid, $deviceid); };
4060         if (my $err = $@) {
4061             eval { qemu_drivedel($vmid, $deviceid); };
4062             warn $@ if $@;
4063             die $err;
4064         }
4065     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4066         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4067         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4068         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4069
4070         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4071
4072         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4073             qemu_iothread_add($vmid, $deviceid, $device);
4074             $devicefull .= ",iothread=iothread-$deviceid";
4075         }
4076
4077         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4078             $devicefull .= ",num_queues=$device->{queues}";
4079         }
4080
4081         qemu_deviceadd($vmid, $devicefull);
4082         qemu_deviceaddverify($vmid, $deviceid);
4083     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4084         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4085         qemu_driveadd($storecfg, $vmid, $device);
4086
4087         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4088         eval { qemu_deviceadd($vmid, $devicefull); };
4089         if (my $err = $@) {
4090             eval { qemu_drivedel($vmid, $deviceid); };
4091             warn $@ if $@;
4092             die $err;
4093         }
4094     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4095         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4096
4097         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4098         my $use_old_bios_files = undef;
4099         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4100
4101         my $netdevicefull = print_netdevice_full(
4102             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4103         qemu_deviceadd($vmid, $netdevicefull);
4104         eval {
4105             qemu_deviceaddverify($vmid, $deviceid);
4106             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4107         };
4108         if (my $err = $@) {
4109             eval { qemu_netdevdel($vmid, $deviceid); };
4110             warn $@ if $@;
4111             die $err;
4112         }
4113     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4114         my $bridgeid = $2;
4115         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4116         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4117
4118         qemu_deviceadd($vmid, $devicefull);
4119         qemu_deviceaddverify($vmid, $deviceid);
4120     } else {
4121         die "can't hotplug device '$deviceid'\n";
4122     }
4123
4124     return 1;
4125 }
4126
4127 # fixme: this should raise exceptions on error!
4128 sub vm_deviceunplug {
4129     my ($vmid, $conf, $deviceid) = @_;
4130
4131     my $devices_list = vm_devices_list($vmid);
4132     return 1 if !defined($devices_list->{$deviceid});
4133
4134     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4135     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4136
4137     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4138         qemu_devicedel($vmid, $deviceid);
4139     } elsif ($deviceid =~ m/^usb\d+$/) {
4140         die "usb hotplug currently not reliable\n";
4141         # when unplugging usb devices this way, there may be remaining usb
4142         # controllers/hubs so we disable it for now
4143         #qemu_devicedel($vmid, $deviceid);
4144         #qemu_devicedelverify($vmid, $deviceid);
4145     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4146         my $device = parse_drive($deviceid, $conf->{$deviceid});
4147
4148         qemu_devicedel($vmid, $deviceid);
4149         qemu_devicedelverify($vmid, $deviceid);
4150         qemu_drivedel($vmid, $deviceid);
4151         qemu_iothread_del($vmid, $deviceid, $device);
4152     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4153         qemu_devicedel($vmid, $deviceid);
4154         qemu_devicedelverify($vmid, $deviceid);
4155     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4156         my $device = parse_drive($deviceid, $conf->{$deviceid});
4157
4158         qemu_devicedel($vmid, $deviceid);
4159         qemu_drivedel($vmid, $deviceid);
4160         qemu_deletescsihw($conf, $vmid, $deviceid);
4161
4162         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4163             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4164     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4165         qemu_devicedel($vmid, $deviceid);
4166         qemu_devicedelverify($vmid, $deviceid);
4167         qemu_netdevdel($vmid, $deviceid);
4168     } else {
4169         die "can't unplug device '$deviceid'\n";
4170     }
4171
4172     return 1;
4173 }
4174
4175 sub qemu_deviceadd {
4176     my ($vmid, $devicefull) = @_;
4177
4178     $devicefull = "driver=".$devicefull;
4179     my %options =  split(/[=,]/, $devicefull);
4180
4181     mon_cmd($vmid, "device_add" , %options);
4182 }
4183
4184 sub qemu_devicedel {
4185     my ($vmid, $deviceid) = @_;
4186
4187     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4188 }
4189
4190 sub qemu_iothread_add {
4191     my ($vmid, $deviceid, $device) = @_;
4192
4193     if ($device->{iothread}) {
4194         my $iothreads = vm_iothreads_list($vmid);
4195         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4196     }
4197 }
4198
4199 sub qemu_iothread_del {
4200     my ($vmid, $deviceid, $device) = @_;
4201
4202     if ($device->{iothread}) {
4203         my $iothreads = vm_iothreads_list($vmid);
4204         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4205     }
4206 }
4207
4208 sub qemu_objectadd {
4209     my ($vmid, $objectid, $qomtype) = @_;
4210
4211     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4212
4213     return 1;
4214 }
4215
4216 sub qemu_objectdel {
4217     my ($vmid, $objectid) = @_;
4218
4219     mon_cmd($vmid, "object-del", id => $objectid);
4220
4221     return 1;
4222 }
4223
4224 sub qemu_driveadd {
4225     my ($storecfg, $vmid, $device) = @_;
4226
4227     my $kvmver = get_running_qemu_version($vmid);
4228     my $io_uring = min_version($kvmver, 6, 0);
4229     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4230     $drive =~ s/\\/\\\\/g;
4231     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4232
4233     # If the command succeeds qemu prints: "OK"
4234     return 1 if $ret =~ m/OK/s;
4235
4236     die "adding drive failed: $ret\n";
4237 }
4238
4239 sub qemu_drivedel {
4240     my ($vmid, $deviceid) = @_;
4241
4242     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4243     $ret =~ s/^\s+//;
4244
4245     return 1 if $ret eq "";
4246
4247     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4248     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4249
4250     die "deleting drive $deviceid failed : $ret\n";
4251 }
4252
4253 sub qemu_deviceaddverify {
4254     my ($vmid, $deviceid) = @_;
4255
4256     for (my $i = 0; $i <= 5; $i++) {
4257          my $devices_list = vm_devices_list($vmid);
4258          return 1 if defined($devices_list->{$deviceid});
4259          sleep 1;
4260     }
4261
4262     die "error on hotplug device '$deviceid'\n";
4263 }
4264
4265
4266 sub qemu_devicedelverify {
4267     my ($vmid, $deviceid) = @_;
4268
4269     # need to verify that the device is correctly removed as device_del
4270     # is async and empty return is not reliable
4271
4272     for (my $i = 0; $i <= 5; $i++) {
4273          my $devices_list = vm_devices_list($vmid);
4274          return 1 if !defined($devices_list->{$deviceid});
4275          sleep 1;
4276     }
4277
4278     die "error on hot-unplugging device '$deviceid'\n";
4279 }
4280
4281 sub qemu_findorcreatescsihw {
4282     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4283
4284     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4285
4286     my $scsihwid="$controller_prefix$controller";
4287     my $devices_list = vm_devices_list($vmid);
4288
4289     if (!defined($devices_list->{$scsihwid})) {
4290         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4291     }
4292
4293     return 1;
4294 }
4295
4296 sub qemu_deletescsihw {
4297     my ($conf, $vmid, $opt) = @_;
4298
4299     my $device = parse_drive($opt, $conf->{$opt});
4300
4301     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4302         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4303         return 1;
4304     }
4305
4306     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4307
4308     my $devices_list = vm_devices_list($vmid);
4309     foreach my $opt (keys %{$devices_list}) {
4310         if (is_valid_drivename($opt)) {
4311             my $drive = parse_drive($opt, $conf->{$opt});
4312             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4313                 return 1;
4314             }
4315         }
4316     }
4317
4318     my $scsihwid="scsihw$controller";
4319
4320     vm_deviceunplug($vmid, $conf, $scsihwid);
4321
4322     return 1;
4323 }
4324
4325 sub qemu_add_pci_bridge {
4326     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4327
4328     my $bridges = {};
4329
4330     my $bridgeid;
4331
4332     print_pci_addr($device, $bridges, $arch, $machine_type);
4333
4334     while (my ($k, $v) = each %$bridges) {
4335         $bridgeid = $k;
4336     }
4337     return 1 if !defined($bridgeid) || $bridgeid < 1;
4338
4339     my $bridge = "pci.$bridgeid";
4340     my $devices_list = vm_devices_list($vmid);
4341
4342     if (!defined($devices_list->{$bridge})) {
4343         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4344     }
4345
4346     return 1;
4347 }
4348
4349 sub qemu_set_link_status {
4350     my ($vmid, $device, $up) = @_;
4351
4352     mon_cmd($vmid, "set_link", name => $device,
4353                up => $up ? JSON::true : JSON::false);
4354 }
4355
4356 sub qemu_netdevadd {
4357     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4358
4359     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4360     my %options =  split(/[=,]/, $netdev);
4361
4362     if (defined(my $vhost = $options{vhost})) {
4363         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4364     }
4365
4366     if (defined(my $queues = $options{queues})) {
4367         $options{queues} = $queues + 0;
4368     }
4369
4370     mon_cmd($vmid, "netdev_add",  %options);
4371     return 1;
4372 }
4373
4374 sub qemu_netdevdel {
4375     my ($vmid, $deviceid) = @_;
4376
4377     mon_cmd($vmid, "netdev_del", id => $deviceid);
4378 }
4379
4380 sub qemu_usb_hotplug {
4381     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4382
4383     return if !$device;
4384
4385     # remove the old one first
4386     vm_deviceunplug($vmid, $conf, $deviceid);
4387
4388     # check if xhci controller is necessary and available
4389     if ($device->{usb3}) {
4390
4391         my $devicelist = vm_devices_list($vmid);
4392
4393         if (!$devicelist->{xhci}) {
4394             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4395             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4396         }
4397     }
4398     my $d = parse_usb_device($device->{host});
4399     $d->{usb3} = $device->{usb3};
4400
4401     # add the new one
4402     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4403 }
4404
4405 sub qemu_cpu_hotplug {
4406     my ($vmid, $conf, $vcpus) = @_;
4407
4408     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4409
4410     my $sockets = 1;
4411     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4412     $sockets = $conf->{sockets} if  $conf->{sockets};
4413     my $cores = $conf->{cores} || 1;
4414     my $maxcpus = $sockets * $cores;
4415
4416     $vcpus = $maxcpus if !$vcpus;
4417
4418     die "you can't add more vcpus than maxcpus\n"
4419         if $vcpus > $maxcpus;
4420
4421     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4422
4423     if ($vcpus < $currentvcpus) {
4424
4425         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4426
4427             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4428                 qemu_devicedel($vmid, "cpu$i");
4429                 my $retry = 0;
4430                 my $currentrunningvcpus = undef;
4431                 while (1) {
4432                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4433                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4434                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4435                     $retry++;
4436                     sleep 1;
4437                 }
4438                 #update conf after each succesfull cpu unplug
4439                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4440                 PVE::QemuConfig->write_config($vmid, $conf);
4441             }
4442         } else {
4443             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4444         }
4445
4446         return;
4447     }
4448
4449     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4450     die "vcpus in running vm does not match its configuration\n"
4451         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4452
4453     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4454
4455         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4456             my $cpustr = print_cpu_device($conf, $i);
4457             qemu_deviceadd($vmid, $cpustr);
4458
4459             my $retry = 0;
4460             my $currentrunningvcpus = undef;
4461             while (1) {
4462                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4463                 last if scalar(@{$currentrunningvcpus}) == $i;
4464                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4465                 sleep 1;
4466                 $retry++;
4467             }
4468             #update conf after each succesfull cpu hotplug
4469             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4470             PVE::QemuConfig->write_config($vmid, $conf);
4471         }
4472     } else {
4473
4474         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4475             mon_cmd($vmid, "cpu-add", id => int($i));
4476         }
4477     }
4478 }
4479
4480 sub qemu_block_set_io_throttle {
4481     my ($vmid, $deviceid,
4482         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4483         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4484         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4485         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4486
4487     return if !check_running($vmid) ;
4488
4489     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4490         bps => int($bps),
4491         bps_rd => int($bps_rd),
4492         bps_wr => int($bps_wr),
4493         iops => int($iops),
4494         iops_rd => int($iops_rd),
4495         iops_wr => int($iops_wr),
4496         bps_max => int($bps_max),
4497         bps_rd_max => int($bps_rd_max),
4498         bps_wr_max => int($bps_wr_max),
4499         iops_max => int($iops_max),
4500         iops_rd_max => int($iops_rd_max),
4501         iops_wr_max => int($iops_wr_max),
4502         bps_max_length => int($bps_max_length),
4503         bps_rd_max_length => int($bps_rd_max_length),
4504         bps_wr_max_length => int($bps_wr_max_length),
4505         iops_max_length => int($iops_max_length),
4506         iops_rd_max_length => int($iops_rd_max_length),
4507         iops_wr_max_length => int($iops_wr_max_length),
4508     );
4509
4510 }
4511
4512 sub qemu_block_resize {
4513     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4514
4515     my $running = check_running($vmid);
4516
4517     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4518
4519     return if !$running;
4520
4521     my $padding = (1024 - $size % 1024) % 1024;
4522     $size = $size + $padding;
4523
4524     mon_cmd(
4525         $vmid,
4526         "block_resize",
4527         device => $deviceid,
4528         size => int($size),
4529         timeout => 60,
4530     );
4531 }
4532
4533 sub qemu_volume_snapshot {
4534     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4535
4536     my $running = check_running($vmid);
4537
4538     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4539         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4540     } else {
4541         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4542     }
4543 }
4544
4545 sub qemu_volume_snapshot_delete {
4546     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4547
4548     my $running = check_running($vmid);
4549
4550     if($running) {
4551
4552         $running = undef;
4553         my $conf = PVE::QemuConfig->load_config($vmid);
4554         PVE::QemuConfig->foreach_volume($conf, sub {
4555             my ($ds, $drive) = @_;
4556             $running = 1 if $drive->{file} eq $volid;
4557         });
4558     }
4559
4560     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4561         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4562     } else {
4563         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4564     }
4565 }
4566
4567 sub set_migration_caps {
4568     my ($vmid, $savevm) = @_;
4569
4570     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4571
4572     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4573     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4574
4575     my $cap_ref = [];
4576
4577     my $enabled_cap = {
4578         "auto-converge" => 1,
4579         "xbzrle" => 1,
4580         "x-rdma-pin-all" => 0,
4581         "zero-blocks" => 0,
4582         "compress" => 0,
4583         "dirty-bitmaps" => $dirty_bitmaps,
4584     };
4585
4586     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4587
4588     for my $supported_capability (@$supported_capabilities) {
4589         push @$cap_ref, {
4590             capability => $supported_capability->{capability},
4591             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4592         };
4593     }
4594
4595     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4596 }
4597
4598 sub foreach_volid {
4599     my ($conf, $func, @param) = @_;
4600
4601     my $volhash = {};
4602
4603     my $test_volid = sub {
4604         my ($key, $drive, $snapname) = @_;
4605
4606         my $volid = $drive->{file};
4607         return if !$volid;
4608
4609         $volhash->{$volid}->{cdrom} //= 1;
4610         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4611
4612         my $replicate = $drive->{replicate} // 1;
4613         $volhash->{$volid}->{replicate} //= 0;
4614         $volhash->{$volid}->{replicate} = 1 if $replicate;
4615
4616         $volhash->{$volid}->{shared} //= 0;
4617         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4618
4619         $volhash->{$volid}->{referenced_in_config} //= 0;
4620         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4621
4622         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4623             if defined($snapname);
4624
4625         my $size = $drive->{size};
4626         $volhash->{$volid}->{size} //= $size if $size;
4627
4628         $volhash->{$volid}->{is_vmstate} //= 0;
4629         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4630
4631         $volhash->{$volid}->{is_tpmstate} //= 0;
4632         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4633
4634         $volhash->{$volid}->{is_unused} //= 0;
4635         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4636
4637         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4638     };
4639
4640     my $include_opts = {
4641         extra_keys => ['vmstate'],
4642         include_unused => 1,
4643     };
4644
4645     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4646     foreach my $snapname (keys %{$conf->{snapshots}}) {
4647         my $snap = $conf->{snapshots}->{$snapname};
4648         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4649     }
4650
4651     foreach my $volid (keys %$volhash) {
4652         &$func($volid, $volhash->{$volid}, @param);
4653     }
4654 }
4655
4656 my $fast_plug_option = {
4657     'lock' => 1,
4658     'name' => 1,
4659     'onboot' => 1,
4660     'shares' => 1,
4661     'startup' => 1,
4662     'description' => 1,
4663     'protection' => 1,
4664     'vmstatestorage' => 1,
4665     'hookscript' => 1,
4666     'tags' => 1,
4667 };
4668
4669 # hotplug changes in [PENDING]
4670 # $selection hash can be used to only apply specified options, for
4671 # example: { cores => 1 } (only apply changed 'cores')
4672 # $errors ref is used to return error messages
4673 sub vmconfig_hotplug_pending {
4674     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4675
4676     my $defaults = load_defaults();
4677     my $arch = get_vm_arch($conf);
4678     my $machine_type = get_vm_machine($conf, undef, $arch);
4679
4680     # commit values which do not have any impact on running VM first
4681     # Note: those option cannot raise errors, we we do not care about
4682     # $selection and always apply them.
4683
4684     my $add_error = sub {
4685         my ($opt, $msg) = @_;
4686         $errors->{$opt} = "hotplug problem - $msg";
4687     };
4688
4689     my $changes = 0;
4690     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4691         if ($fast_plug_option->{$opt}) {
4692             $conf->{$opt} = $conf->{pending}->{$opt};
4693             delete $conf->{pending}->{$opt};
4694             $changes = 1;
4695         }
4696     }
4697
4698     if ($changes) {
4699         PVE::QemuConfig->write_config($vmid, $conf);
4700     }
4701
4702     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4703
4704     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4705     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4706     foreach my $opt (sort keys %$pending_delete_hash) {
4707         next if $selection && !$selection->{$opt};
4708         my $force = $pending_delete_hash->{$opt}->{force};
4709         eval {
4710             if ($opt eq 'hotplug') {
4711                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4712             } elsif ($opt eq 'tablet') {
4713                 die "skip\n" if !$hotplug_features->{usb};
4714                 if ($defaults->{tablet}) {
4715                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4716                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4717                         if $arch eq 'aarch64';
4718                 } else {
4719                     vm_deviceunplug($vmid, $conf, 'tablet');
4720                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4721                 }
4722             } elsif ($opt =~ m/^usb\d+/) {
4723                 die "skip\n";
4724                 # since we cannot reliably hot unplug usb devices we are disabling it
4725                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4726                 #vm_deviceunplug($vmid, $conf, $opt);
4727             } elsif ($opt eq 'vcpus') {
4728                 die "skip\n" if !$hotplug_features->{cpu};
4729                 qemu_cpu_hotplug($vmid, $conf, undef);
4730             } elsif ($opt eq 'balloon') {
4731                 # enable balloon device is not hotpluggable
4732                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4733                 # here we reset the ballooning value to memory
4734                 my $balloon = $conf->{memory} || $defaults->{memory};
4735                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4736             } elsif ($fast_plug_option->{$opt}) {
4737                 # do nothing
4738             } elsif ($opt =~ m/^net(\d+)$/) {
4739                 die "skip\n" if !$hotplug_features->{network};
4740                 vm_deviceunplug($vmid, $conf, $opt);
4741             } elsif (is_valid_drivename($opt)) {
4742                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4743                 vm_deviceunplug($vmid, $conf, $opt);
4744                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4745             } elsif ($opt =~ m/^memory$/) {
4746                 die "skip\n" if !$hotplug_features->{memory};
4747                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4748             } elsif ($opt eq 'cpuunits') {
4749                 $cgroup->change_cpu_shares(undef, 1024);
4750             } elsif ($opt eq 'cpulimit') {
4751                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4752             } else {
4753                 die "skip\n";
4754             }
4755         };
4756         if (my $err = $@) {
4757             &$add_error($opt, $err) if $err ne "skip\n";
4758         } else {
4759             delete $conf->{$opt};
4760             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4761         }
4762     }
4763
4764     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4765     $apply_pending_cloudinit = sub {
4766         return if $apply_pending_cloudinit_done; # once is enough
4767         $apply_pending_cloudinit_done = 1; # once is enough
4768
4769         my ($key, $value) = @_;
4770
4771         my @cloudinit_opts = keys %$confdesc_cloudinit;
4772         foreach my $opt (keys %{$conf->{pending}}) {
4773             next if !grep { $_ eq $opt } @cloudinit_opts;
4774             $conf->{$opt} = delete $conf->{pending}->{$opt};
4775         }
4776
4777         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4778         foreach my $opt (sort keys %$pending_delete_hash) {
4779             next if !grep { $_ eq $opt } @cloudinit_opts;
4780             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4781             delete $conf->{$opt};
4782         }
4783
4784         my $new_conf = { %$conf };
4785         $new_conf->{$key} = $value;
4786         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4787     };
4788
4789     foreach my $opt (keys %{$conf->{pending}}) {
4790         next if $selection && !$selection->{$opt};
4791         my $value = $conf->{pending}->{$opt};
4792         eval {
4793             if ($opt eq 'hotplug') {
4794                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4795             } elsif ($opt eq 'tablet') {
4796                 die "skip\n" if !$hotplug_features->{usb};
4797                 if ($value == 1) {
4798                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4799                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4800                         if $arch eq 'aarch64';
4801                 } elsif ($value == 0) {
4802                     vm_deviceunplug($vmid, $conf, 'tablet');
4803                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4804                 }
4805             } elsif ($opt =~ m/^usb\d+$/) {
4806                 die "skip\n";
4807                 # since we cannot reliably hot unplug usb devices we disable it for now
4808                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4809                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4810                 #die "skip\n" if !$d;
4811                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4812             } elsif ($opt eq 'vcpus') {
4813                 die "skip\n" if !$hotplug_features->{cpu};
4814                 qemu_cpu_hotplug($vmid, $conf, $value);
4815             } elsif ($opt eq 'balloon') {
4816                 # enable/disable balloning device is not hotpluggable
4817                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4818                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4819                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4820
4821                 # allow manual ballooning if shares is set to zero
4822                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4823                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4824                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4825                 }
4826             } elsif ($opt =~ m/^net(\d+)$/) {
4827                 # some changes can be done without hotplug
4828                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4829                                     $vmid, $opt, $value, $arch, $machine_type);
4830             } elsif (is_valid_drivename($opt)) {
4831                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4832                 # some changes can be done without hotplug
4833                 my $drive = parse_drive($opt, $value);
4834                 if (drive_is_cloudinit($drive)) {
4835                     &$apply_pending_cloudinit($opt, $value);
4836                 }
4837                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4838                                      $vmid, $opt, $value, $arch, $machine_type);
4839             } elsif ($opt =~ m/^memory$/) { #dimms
4840                 die "skip\n" if !$hotplug_features->{memory};
4841                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4842             } elsif ($opt eq 'cpuunits') {
4843                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4844             } elsif ($opt eq 'cpulimit') {
4845                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4846                 $cgroup->change_cpu_quota($cpulimit, 100000);
4847             } else {
4848                 die "skip\n";  # skip non-hot-pluggable options
4849             }
4850         };
4851         if (my $err = $@) {
4852             &$add_error($opt, $err) if $err ne "skip\n";
4853         } else {
4854             $conf->{$opt} = $value;
4855             delete $conf->{pending}->{$opt};
4856         }
4857     }
4858
4859     PVE::QemuConfig->write_config($vmid, $conf);
4860 }
4861
4862 sub try_deallocate_drive {
4863     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4864
4865     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4866         my $volid = $drive->{file};
4867         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4868             my $sid = PVE::Storage::parse_volume_id($volid);
4869             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4870
4871             # check if the disk is really unused
4872             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4873                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4874             PVE::Storage::vdisk_free($storecfg, $volid);
4875             return 1;
4876         } else {
4877             # If vm is not owner of this disk remove from config
4878             return 1;
4879         }
4880     }
4881
4882     return;
4883 }
4884
4885 sub vmconfig_delete_or_detach_drive {
4886     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4887
4888     my $drive = parse_drive($opt, $conf->{$opt});
4889
4890     my $rpcenv = PVE::RPCEnvironment::get();
4891     my $authuser = $rpcenv->get_user();
4892
4893     if ($force) {
4894         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4895         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4896     } else {
4897         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4898     }
4899 }
4900
4901
4902
4903 sub vmconfig_apply_pending {
4904     my ($vmid, $conf, $storecfg, $errors) = @_;
4905
4906     my $add_apply_error = sub {
4907         my ($opt, $msg) = @_;
4908         my $err_msg = "unable to apply pending change $opt : $msg";
4909         $errors->{$opt} = $err_msg;
4910         warn $err_msg;
4911     };
4912
4913     # cold plug
4914
4915     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4916     foreach my $opt (sort keys %$pending_delete_hash) {
4917         my $force = $pending_delete_hash->{$opt}->{force};
4918         eval {
4919             if ($opt =~ m/^unused/) {
4920                 die "internal error";
4921             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4922                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4923             }
4924         };
4925         if (my $err = $@) {
4926             $add_apply_error->($opt, $err);
4927         } else {
4928             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4929             delete $conf->{$opt};
4930         }
4931     }
4932
4933     PVE::QemuConfig->cleanup_pending($conf);
4934
4935     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4936         next if $opt eq 'delete'; # just to be sure
4937         eval {
4938             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4939                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4940             }
4941         };
4942         if (my $err = $@) {
4943             $add_apply_error->($opt, $err);
4944         } else {
4945             $conf->{$opt} = delete $conf->{pending}->{$opt};
4946         }
4947     }
4948
4949     # write all changes at once to avoid unnecessary i/o
4950     PVE::QemuConfig->write_config($vmid, $conf);
4951 }
4952
4953 sub vmconfig_update_net {
4954     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4955
4956     my $newnet = parse_net($value);
4957
4958     if ($conf->{$opt}) {
4959         my $oldnet = parse_net($conf->{$opt});
4960
4961         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4962             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4963             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4964             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4965
4966             # for non online change, we try to hot-unplug
4967             die "skip\n" if !$hotplug;
4968             vm_deviceunplug($vmid, $conf, $opt);
4969         } else {
4970
4971             die "internal error" if $opt !~ m/net(\d+)/;
4972             my $iface = "tap${vmid}i$1";
4973
4974             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4975                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4976                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4977                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4978                 PVE::Network::tap_unplug($iface);
4979
4980                 if ($have_sdn) {
4981                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4982                 } else {
4983                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4984                 }
4985             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4986                 # Rate can be applied on its own but any change above needs to
4987                 # include the rate in tap_plug since OVS resets everything.
4988                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4989             }
4990
4991             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4992                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4993             }
4994
4995             return 1;
4996         }
4997     }
4998
4999     if ($hotplug) {
5000         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5001     } else {
5002         die "skip\n";
5003     }
5004 }
5005
5006 sub vmconfig_update_disk {
5007     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5008
5009     my $drive = parse_drive($opt, $value);
5010
5011     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5012         my $media = $drive->{media} || 'disk';
5013         my $oldmedia = $old_drive->{media} || 'disk';
5014         die "unable to change media type\n" if $media ne $oldmedia;
5015
5016         if (!drive_is_cdrom($old_drive)) {
5017
5018             if ($drive->{file} ne $old_drive->{file}) {
5019
5020                 die "skip\n" if !$hotplug;
5021
5022                 # unplug and register as unused
5023                 vm_deviceunplug($vmid, $conf, $opt);
5024                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5025
5026             } else {
5027                 # update existing disk
5028
5029                 # skip non hotpluggable value
5030                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5031                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5032                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5033                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5034                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5035                     die "skip\n";
5036                 }
5037
5038                 # apply throttle
5039                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5040                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5041                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5042                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5043                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5044                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5045                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5046                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5047                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5048                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5049                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5050                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5051                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5052                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5053                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5054                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5055                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5056                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5057
5058                     qemu_block_set_io_throttle(
5059                         $vmid,"drive-$opt",
5060                         ($drive->{mbps} || 0)*1024*1024,
5061                         ($drive->{mbps_rd} || 0)*1024*1024,
5062                         ($drive->{mbps_wr} || 0)*1024*1024,
5063                          $drive->{iops} || 0,
5064                          $drive->{iops_rd} || 0,
5065                          $drive->{iops_wr} || 0,
5066                         ($drive->{mbps_max} || 0)*1024*1024,
5067                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5068                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5069                          $drive->{iops_max} || 0,
5070                          $drive->{iops_rd_max} || 0,
5071                          $drive->{iops_wr_max} || 0,
5072                          $drive->{bps_max_length} || 1,
5073                          $drive->{bps_rd_max_length} || 1,
5074                          $drive->{bps_wr_max_length} || 1,
5075                          $drive->{iops_max_length} || 1,
5076                          $drive->{iops_rd_max_length} || 1,
5077                          $drive->{iops_wr_max_length} || 1,
5078                     );
5079
5080                 }
5081
5082                 return 1;
5083             }
5084
5085         } else { # cdrom
5086
5087             if ($drive->{file} eq 'none') {
5088                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5089                 if (drive_is_cloudinit($old_drive)) {
5090                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5091                 }
5092             } else {
5093                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5094
5095                 # force eject if locked
5096                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5097
5098                 if ($path) {
5099                     mon_cmd($vmid, "blockdev-change-medium",
5100                         id => "$opt", filename => "$path");
5101                 }
5102             }
5103
5104             return 1;
5105         }
5106     }
5107
5108     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5109     # hotplug new disks
5110     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5111     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5112 }
5113
5114 # called in locked context by incoming migration
5115 sub vm_migrate_get_nbd_disks {
5116     my ($storecfg, $conf, $replicated_volumes) = @_;
5117
5118     my $local_volumes = {};
5119     PVE::QemuConfig->foreach_volume($conf, sub {
5120         my ($ds, $drive) = @_;
5121
5122         return if drive_is_cdrom($drive);
5123
5124         my $volid = $drive->{file};
5125
5126         return if !$volid;
5127
5128         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5129
5130         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5131         return if $scfg->{shared};
5132
5133         # replicated disks re-use existing state via bitmap
5134         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5135         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5136     });
5137     return $local_volumes;
5138 }
5139
5140 # called in locked context by incoming migration
5141 sub vm_migrate_alloc_nbd_disks {
5142     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5143
5144     my $format = undef;
5145
5146     my $nbd = {};
5147     foreach my $opt (sort keys %$source_volumes) {
5148         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5149
5150         if ($use_existing) {
5151             $nbd->{$opt}->{drivestr} = print_drive($drive);
5152             $nbd->{$opt}->{volid} = $volid;
5153             $nbd->{$opt}->{replicated} = 1;
5154             next;
5155         }
5156
5157         # If a remote storage is specified and the format of the original
5158         # volume is not available there, fall back to the default format.
5159         # Otherwise use the same format as the original.
5160         if (!$storagemap->{identity}) {
5161             $storeid = map_storage($storagemap, $storeid);
5162             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5163             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5164             my $fileFormat = qemu_img_format($scfg, $volname);
5165             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5166         } else {
5167             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5168             $format = qemu_img_format($scfg, $volname);
5169         }
5170
5171         my $size = $drive->{size} / 1024;
5172         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5173         my $newdrive = $drive;
5174         $newdrive->{format} = $format;
5175         $newdrive->{file} = $newvolid;
5176         my $drivestr = print_drive($newdrive);
5177         $nbd->{$opt}->{drivestr} = $drivestr;
5178         $nbd->{$opt}->{volid} = $newvolid;
5179     }
5180
5181     return $nbd;
5182 }
5183
5184 # see vm_start_nolock for parameters, additionally:
5185 # migrate_opts:
5186 #   storagemap = parsed storage map for allocating NBD disks
5187 sub vm_start {
5188     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5189
5190     return PVE::QemuConfig->lock_config($vmid, sub {
5191         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5192
5193         die "you can't start a vm if it's a template\n"
5194             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5195
5196         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5197         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5198
5199         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5200
5201         if ($has_backup_lock && $running) {
5202             # a backup is currently running, attempt to start the guest in the
5203             # existing QEMU instance
5204             return vm_resume($vmid);
5205         }
5206
5207         PVE::QemuConfig->check_lock($conf)
5208             if !($params->{skiplock} || $has_suspended_lock);
5209
5210         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5211
5212         die "VM $vmid already running\n" if $running;
5213
5214         if (my $storagemap = $migrate_opts->{storagemap}) {
5215             my $replicated = $migrate_opts->{replicated_volumes};
5216             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5217             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5218
5219             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5220                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5221             }
5222         }
5223
5224         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5225     });
5226 }
5227
5228
5229 # params:
5230 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5231 #   skiplock => 0/1, skip checking for config lock
5232 #   skiptemplate => 0/1, skip checking whether VM is template
5233 #   forcemachine => to force Qemu machine (rollback/migration)
5234 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5235 #   timeout => in seconds
5236 #   paused => start VM in paused state (backup)
5237 #   resume => resume from hibernation
5238 #   pbs-backing => {
5239 #      sata0 => {
5240 #         repository
5241 #         snapshot
5242 #         keyfile
5243 #         archive
5244 #      },
5245 #      virtio2 => ...
5246 #   }
5247 # migrate_opts:
5248 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5249 #   migratedfrom => source node
5250 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5251 #   network => CIDR of migration network
5252 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5253 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5254 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5255 sub vm_start_nolock {
5256     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5257
5258     my $statefile = $params->{statefile};
5259     my $resume = $params->{resume};
5260
5261     my $migratedfrom = $migrate_opts->{migratedfrom};
5262     my $migration_type = $migrate_opts->{type};
5263
5264     my $res = {};
5265
5266     # clean up leftover reboot request files
5267     eval { clear_reboot_request($vmid); };
5268     warn $@ if $@;
5269
5270     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5271         vmconfig_apply_pending($vmid, $conf, $storecfg);
5272         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5273     }
5274
5275     # don't regenerate the ISO if the VM is started as part of a live migration
5276     # this way we can reuse the old ISO with the correct config
5277     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5278
5279     my $defaults = load_defaults();
5280
5281     # set environment variable useful inside network script
5282     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5283
5284     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5285
5286     my $forcemachine = $params->{forcemachine};
5287     my $forcecpu = $params->{forcecpu};
5288     if ($resume) {
5289         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5290         $forcemachine = $conf->{runningmachine};
5291         $forcecpu = $conf->{runningcpu};
5292         print "Resuming suspended VM\n";
5293     }
5294
5295     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5296         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5297
5298     my $migration_ip;
5299     my $get_migration_ip = sub {
5300         my ($nodename) = @_;
5301
5302         return $migration_ip if defined($migration_ip);
5303
5304         my $cidr = $migrate_opts->{network};
5305
5306         if (!defined($cidr)) {
5307             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5308             $cidr = $dc_conf->{migration}->{network};
5309         }
5310
5311         if (defined($cidr)) {
5312             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5313
5314             die "could not get IP: no address configured on local " .
5315                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5316
5317             die "could not get IP: multiple addresses configured on local " .
5318                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5319
5320             $migration_ip = @$ips[0];
5321         }
5322
5323         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5324             if !defined($migration_ip);
5325
5326         return $migration_ip;
5327     };
5328
5329     my $migrate_uri;
5330     if ($statefile) {
5331         if ($statefile eq 'tcp') {
5332             my $localip = "localhost";
5333             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5334             my $nodename = nodename();
5335
5336             if (!defined($migration_type)) {
5337                 if (defined($datacenterconf->{migration}->{type})) {
5338                     $migration_type = $datacenterconf->{migration}->{type};
5339                 } else {
5340                     $migration_type = 'secure';
5341                 }
5342             }
5343
5344             if ($migration_type eq 'insecure') {
5345                 $localip = $get_migration_ip->($nodename);
5346                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5347             }
5348
5349             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5350             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5351             $migrate_uri = "tcp:${localip}:${migrate_port}";
5352             push @$cmd, '-incoming', $migrate_uri;
5353             push @$cmd, '-S';
5354
5355         } elsif ($statefile eq 'unix') {
5356             # should be default for secure migrations as a ssh TCP forward
5357             # tunnel is not deterministic reliable ready and fails regurarly
5358             # to set up in time, so use UNIX socket forwards
5359             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5360             unlink $socket_addr;
5361
5362             $migrate_uri = "unix:$socket_addr";
5363
5364             push @$cmd, '-incoming', $migrate_uri;
5365             push @$cmd, '-S';
5366
5367         } elsif (-e $statefile) {
5368             push @$cmd, '-loadstate', $statefile;
5369         } else {
5370             my $statepath = PVE::Storage::path($storecfg, $statefile);
5371             push @$vollist, $statefile;
5372             push @$cmd, '-loadstate', $statepath;
5373         }
5374     } elsif ($params->{paused}) {
5375         push @$cmd, '-S';
5376     }
5377
5378     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5379
5380     my $pci_devices = {}; # host pci devices
5381     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5382         my $dev = $conf->{"hostpci$i"} or next;
5383         $pci_devices->{$i} = parse_hostpci($dev);
5384     }
5385
5386     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } values $pci_devices->%* ];
5387     # reserve all PCI IDs before actually doing anything with them
5388     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5389
5390     eval {
5391         for my $id (sort keys %$pci_devices) {
5392             my $d = $pci_devices->{$id};
5393             for my $dev ($d->{pciid}->@*) {
5394                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5395             }
5396         }
5397     };
5398     if (my $err = $@) {
5399         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5400         warn $@ if $@;
5401         die $err;
5402     }
5403
5404     PVE::Storage::activate_volumes($storecfg, $vollist);
5405
5406     eval {
5407         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5408     };
5409     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5410     # timeout should be more than enough here...
5411     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5412
5413     my $cpuunits = get_cpuunits($conf);
5414
5415     my %run_params = (
5416         timeout => $statefile ? undef : $start_timeout,
5417         umask => 0077,
5418         noerr => 1,
5419     );
5420
5421     # when migrating, prefix QEMU output so other side can pick up any
5422     # errors that might occur and show the user
5423     if ($migratedfrom) {
5424         $run_params{quiet} = 1;
5425         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5426     }
5427
5428     my %systemd_properties = (
5429         Slice => 'qemu.slice',
5430         KillMode => 'process',
5431         SendSIGKILL => 0,
5432         TimeoutStopUSec => ULONG_MAX, # infinity
5433     );
5434
5435     if (PVE::CGroup::cgroup_mode() == 2) {
5436         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5437         $systemd_properties{CPUWeight} = $cpuunits;
5438     } else {
5439         $systemd_properties{CPUShares} = $cpuunits;
5440     }
5441
5442     if (my $cpulimit = $conf->{cpulimit}) {
5443         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5444     }
5445     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5446
5447     my $run_qemu = sub {
5448         PVE::Tools::run_fork sub {
5449             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5450
5451             my $tpmpid;
5452             if (my $tpm = $conf->{tpmstate0}) {
5453                 # start the TPM emulator so QEMU can connect on start
5454                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5455             }
5456
5457             my $exitcode = run_command($cmd, %run_params);
5458             if ($exitcode) {
5459                 if ($tpmpid) {
5460                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5461                     kill 'TERM', $tpmpid;
5462                 }
5463                 die "QEMU exited with code $exitcode\n";
5464             }
5465         };
5466     };
5467
5468     if ($conf->{hugepages}) {
5469
5470         my $code = sub {
5471             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5472             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5473
5474             PVE::QemuServer::Memory::hugepages_mount();
5475             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5476
5477             eval { $run_qemu->() };
5478             if (my $err = $@) {
5479                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5480                     if !$conf->{keephugepages};
5481                 die $err;
5482             }
5483
5484             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5485                 if !$conf->{keephugepages};
5486         };
5487         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5488
5489     } else {
5490         eval { $run_qemu->() };
5491     }
5492
5493     if (my $err = $@) {
5494         # deactivate volumes if start fails
5495         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5496         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5497
5498         die "start failed: $err";
5499     }
5500
5501     # re-reserve all PCI IDs now that we can know the actual VM PID
5502     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5503     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5504     warn $@ if $@;
5505
5506     print "migration listens on $migrate_uri\n" if $migrate_uri;
5507     $res->{migrate_uri} = $migrate_uri;
5508
5509     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5510         eval { mon_cmd($vmid, "cont"); };
5511         warn $@ if $@;
5512     }
5513
5514     #start nbd server for storage migration
5515     if (my $nbd = $migrate_opts->{nbd}) {
5516         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5517
5518         my $migrate_storage_uri;
5519         # nbd_protocol_version > 0 for unix socket support
5520         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5521             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5522             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5523             $migrate_storage_uri = "nbd:unix:$socket_path";
5524         } else {
5525             my $nodename = nodename();
5526             my $localip = $get_migration_ip->($nodename);
5527             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5528             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5529
5530             mon_cmd($vmid, "nbd-server-start", addr => {
5531                 type => 'inet',
5532                 data => {
5533                     host => "${localip}",
5534                     port => "${storage_migrate_port}",
5535                 },
5536             });
5537             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5538             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5539         }
5540
5541         $res->{migrate_storage_uri} = $migrate_storage_uri;
5542
5543         foreach my $opt (sort keys %$nbd) {
5544             my $drivestr = $nbd->{$opt}->{drivestr};
5545             my $volid = $nbd->{$opt}->{volid};
5546             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5547             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5548             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5549             print "re-using replicated volume: $opt - $volid\n"
5550                 if $nbd->{$opt}->{replicated};
5551
5552             $res->{drives}->{$opt} = $nbd->{$opt};
5553             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5554         }
5555     }
5556
5557     if ($migratedfrom) {
5558         eval {
5559             set_migration_caps($vmid);
5560         };
5561         warn $@ if $@;
5562
5563         if ($spice_port) {
5564             print "spice listens on port $spice_port\n";
5565             $res->{spice_port} = $spice_port;
5566             if ($migrate_opts->{spice_ticket}) {
5567                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5568                     $migrate_opts->{spice_ticket});
5569                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5570             }
5571         }
5572
5573     } else {
5574         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5575             if !$statefile && $conf->{balloon};
5576
5577         foreach my $opt (keys %$conf) {
5578             next if $opt !~  m/^net\d+$/;
5579             my $nicconf = parse_net($conf->{$opt});
5580             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5581         }
5582     }
5583
5584     mon_cmd($vmid, 'qom-set',
5585                 path => "machine/peripheral/balloon0",
5586                 property => "guest-stats-polling-interval",
5587                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5588
5589     if ($resume) {
5590         print "Resumed VM, removing state\n";
5591         if (my $vmstate = $conf->{vmstate}) {
5592             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5593             PVE::Storage::vdisk_free($storecfg, $vmstate);
5594         }
5595         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5596         PVE::QemuConfig->write_config($vmid, $conf);
5597     }
5598
5599     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5600
5601     return $res;
5602 }
5603
5604 sub vm_commandline {
5605     my ($storecfg, $vmid, $snapname) = @_;
5606
5607     my $conf = PVE::QemuConfig->load_config($vmid);
5608
5609     my ($forcemachine, $forcecpu);
5610     if ($snapname) {
5611         my $snapshot = $conf->{snapshots}->{$snapname};
5612         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5613
5614         # check for machine or CPU overrides in snapshot
5615         $forcemachine = $snapshot->{runningmachine};
5616         $forcecpu = $snapshot->{runningcpu};
5617
5618         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5619
5620         $conf = $snapshot;
5621     }
5622
5623     my $defaults = load_defaults();
5624
5625     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5626
5627     return PVE::Tools::cmd2string($cmd);
5628 }
5629
5630 sub vm_reset {
5631     my ($vmid, $skiplock) = @_;
5632
5633     PVE::QemuConfig->lock_config($vmid, sub {
5634
5635         my $conf = PVE::QemuConfig->load_config($vmid);
5636
5637         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5638
5639         mon_cmd($vmid, "system_reset");
5640     });
5641 }
5642
5643 sub get_vm_volumes {
5644     my ($conf) = @_;
5645
5646     my $vollist = [];
5647     foreach_volid($conf, sub {
5648         my ($volid, $attr) = @_;
5649
5650         return if $volid =~ m|^/|;
5651
5652         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5653         return if !$sid;
5654
5655         push @$vollist, $volid;
5656     });
5657
5658     return $vollist;
5659 }
5660
5661 sub vm_stop_cleanup {
5662     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5663
5664     eval {
5665
5666         if (!$keepActive) {
5667             my $vollist = get_vm_volumes($conf);
5668             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5669
5670             if (my $tpmdrive = $conf->{tpmstate0}) {
5671                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5672                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5673                 if ($storeid) {
5674                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5675                 }
5676             }
5677         }
5678
5679         foreach my $ext (qw(mon qmp pid vnc qga)) {
5680             unlink "/var/run/qemu-server/${vmid}.$ext";
5681         }
5682
5683         if ($conf->{ivshmem}) {
5684             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5685             # just delete it for now, VMs which have this already open do not
5686             # are affected, but new VMs will get a separated one. If this
5687             # becomes an issue we either add some sort of ref-counting or just
5688             # add a "don't delete on stop" flag to the ivshmem format.
5689             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5690         }
5691
5692         my $ids = [];
5693         foreach my $key (keys %$conf) {
5694             next if $key !~ m/^hostpci(\d+)$/;
5695             my $hostpciindex = $1;
5696             my $d = parse_hostpci($conf->{$key});
5697             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5698
5699             foreach my $pci (@{$d->{pciid}}) {
5700                 my $pciid = $pci->{id};
5701                 push @$ids, $pci->{id};
5702                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5703             }
5704         }
5705         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5706
5707         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5708     };
5709     warn $@ if $@; # avoid errors - just warn
5710 }
5711
5712 # call only in locked context
5713 sub _do_vm_stop {
5714     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5715
5716     my $pid = check_running($vmid, $nocheck);
5717     return if !$pid;
5718
5719     my $conf;
5720     if (!$nocheck) {
5721         $conf = PVE::QemuConfig->load_config($vmid);
5722         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5723         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5724             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5725             $timeout = $opts->{down} if $opts->{down};
5726         }
5727         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5728     }
5729
5730     eval {
5731         if ($shutdown) {
5732             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5733                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5734             } else {
5735                 mon_cmd($vmid, "system_powerdown");
5736             }
5737         } else {
5738             mon_cmd($vmid, "quit");
5739         }
5740     };
5741     my $err = $@;
5742
5743     if (!$err) {
5744         $timeout = 60 if !defined($timeout);
5745
5746         my $count = 0;
5747         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5748             $count++;
5749             sleep 1;
5750         }
5751
5752         if ($count >= $timeout) {
5753             if ($force) {
5754                 warn "VM still running - terminating now with SIGTERM\n";
5755                 kill 15, $pid;
5756             } else {
5757                 die "VM quit/powerdown failed - got timeout\n";
5758             }
5759         } else {
5760             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5761             return;
5762         }
5763     } else {
5764        if (!check_running($vmid, $nocheck)) {
5765             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5766             return;
5767        }
5768        if ($force) {
5769             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5770             kill 15, $pid;
5771         } else {
5772             die "VM quit/powerdown failed\n";
5773         }
5774     }
5775
5776     # wait again
5777     $timeout = 10;
5778
5779     my $count = 0;
5780     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5781         $count++;
5782         sleep 1;
5783     }
5784
5785     if ($count >= $timeout) {
5786         warn "VM still running - terminating now with SIGKILL\n";
5787         kill 9, $pid;
5788         sleep 1;
5789     }
5790
5791     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5792 }
5793
5794 # Note: use $nocheck to skip tests if VM configuration file exists.
5795 # We need that when migration VMs to other nodes (files already moved)
5796 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5797 sub vm_stop {
5798     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5799
5800     $force = 1 if !defined($force) && !$shutdown;
5801
5802     if ($migratedfrom){
5803         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5804         kill 15, $pid if $pid;
5805         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5806         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5807         return;
5808     }
5809
5810     PVE::QemuConfig->lock_config($vmid, sub {
5811         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5812    });
5813 }
5814
5815 sub vm_reboot {
5816     my ($vmid, $timeout) = @_;
5817
5818     PVE::QemuConfig->lock_config($vmid, sub {
5819         eval {
5820
5821             # only reboot if running, as qmeventd starts it again on a stop event
5822             return if !check_running($vmid);
5823
5824             create_reboot_request($vmid);
5825
5826             my $storecfg = PVE::Storage::config();
5827             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5828
5829         };
5830         if (my $err = $@) {
5831             # avoid that the next normal shutdown will be confused for a reboot
5832             clear_reboot_request($vmid);
5833             die $err;
5834         }
5835    });
5836 }
5837
5838 # note: if using the statestorage parameter, the caller has to check privileges
5839 sub vm_suspend {
5840     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5841
5842     my $conf;
5843     my $path;
5844     my $storecfg;
5845     my $vmstate;
5846
5847     PVE::QemuConfig->lock_config($vmid, sub {
5848
5849         $conf = PVE::QemuConfig->load_config($vmid);
5850
5851         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5852         PVE::QemuConfig->check_lock($conf)
5853             if !($skiplock || $is_backing_up);
5854
5855         die "cannot suspend to disk during backup\n"
5856             if $is_backing_up && $includestate;
5857
5858         if ($includestate) {
5859             $conf->{lock} = 'suspending';
5860             my $date = strftime("%Y-%m-%d", localtime(time()));
5861             $storecfg = PVE::Storage::config();
5862             if (!$statestorage) {
5863                 $statestorage = find_vmstate_storage($conf, $storecfg);
5864                 # check permissions for the storage
5865                 my $rpcenv = PVE::RPCEnvironment::get();
5866                 if ($rpcenv->{type} ne 'cli') {
5867                     my $authuser = $rpcenv->get_user();
5868                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5869                 }
5870             }
5871
5872
5873             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5874                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5875             $path = PVE::Storage::path($storecfg, $vmstate);
5876             PVE::QemuConfig->write_config($vmid, $conf);
5877         } else {
5878             mon_cmd($vmid, "stop");
5879         }
5880     });
5881
5882     if ($includestate) {
5883         # save vm state
5884         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5885
5886         eval {
5887             set_migration_caps($vmid, 1);
5888             mon_cmd($vmid, "savevm-start", statefile => $path);
5889             for(;;) {
5890                 my $state = mon_cmd($vmid, "query-savevm");
5891                 if (!$state->{status}) {
5892                     die "savevm not active\n";
5893                 } elsif ($state->{status} eq 'active') {
5894                     sleep(1);
5895                     next;
5896                 } elsif ($state->{status} eq 'completed') {
5897                     print "State saved, quitting\n";
5898                     last;
5899                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5900                     die "query-savevm failed with error '$state->{error}'\n"
5901                 } else {
5902                     die "query-savevm returned status '$state->{status}'\n";
5903                 }
5904             }
5905         };
5906         my $err = $@;
5907
5908         PVE::QemuConfig->lock_config($vmid, sub {
5909             $conf = PVE::QemuConfig->load_config($vmid);
5910             if ($err) {
5911                 # cleanup, but leave suspending lock, to indicate something went wrong
5912                 eval {
5913                     mon_cmd($vmid, "savevm-end");
5914                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5915                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5916                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5917                     PVE::QemuConfig->write_config($vmid, $conf);
5918                 };
5919                 warn $@ if $@;
5920                 die $err;
5921             }
5922
5923             die "lock changed unexpectedly\n"
5924                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5925
5926             mon_cmd($vmid, "quit");
5927             $conf->{lock} = 'suspended';
5928             PVE::QemuConfig->write_config($vmid, $conf);
5929         });
5930     }
5931 }
5932
5933 sub vm_resume {
5934     my ($vmid, $skiplock, $nocheck) = @_;
5935
5936     PVE::QemuConfig->lock_config($vmid, sub {
5937         my $res = mon_cmd($vmid, 'query-status');
5938         my $resume_cmd = 'cont';
5939         my $reset = 0;
5940
5941         if ($res->{status}) {
5942             return if $res->{status} eq 'running'; # job done, go home
5943             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5944             $reset = 1 if $res->{status} eq 'shutdown';
5945         }
5946
5947         if (!$nocheck) {
5948
5949             my $conf = PVE::QemuConfig->load_config($vmid);
5950
5951             PVE::QemuConfig->check_lock($conf)
5952                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5953         }
5954
5955         if ($reset) {
5956             # required if a VM shuts down during a backup and we get a resume
5957             # request before the backup finishes for example
5958             mon_cmd($vmid, "system_reset");
5959         }
5960         mon_cmd($vmid, $resume_cmd);
5961     });
5962 }
5963
5964 sub vm_sendkey {
5965     my ($vmid, $skiplock, $key) = @_;
5966
5967     PVE::QemuConfig->lock_config($vmid, sub {
5968
5969         my $conf = PVE::QemuConfig->load_config($vmid);
5970
5971         # there is no qmp command, so we use the human monitor command
5972         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5973         die $res if $res ne '';
5974     });
5975 }
5976
5977 # vzdump restore implementaion
5978
5979 sub tar_archive_read_firstfile {
5980     my $archive = shift;
5981
5982     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5983
5984     # try to detect archive type first
5985     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5986         die "unable to open file '$archive'\n";
5987     my $firstfile = <$fh>;
5988     kill 15, $pid;
5989     close $fh;
5990
5991     die "ERROR: archive contaions no data\n" if !$firstfile;
5992     chomp $firstfile;
5993
5994     return $firstfile;
5995 }
5996
5997 sub tar_restore_cleanup {
5998     my ($storecfg, $statfile) = @_;
5999
6000     print STDERR "starting cleanup\n";
6001
6002     if (my $fd = IO::File->new($statfile, "r")) {
6003         while (defined(my $line = <$fd>)) {
6004             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6005                 my $volid = $2;
6006                 eval {
6007                     if ($volid =~ m|^/|) {
6008                         unlink $volid || die 'unlink failed\n';
6009                     } else {
6010                         PVE::Storage::vdisk_free($storecfg, $volid);
6011                     }
6012                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6013                 };
6014                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6015             } else {
6016                 print STDERR "unable to parse line in statfile - $line";
6017             }
6018         }
6019         $fd->close();
6020     }
6021 }
6022
6023 sub restore_file_archive {
6024     my ($archive, $vmid, $user, $opts) = @_;
6025
6026     return restore_vma_archive($archive, $vmid, $user, $opts)
6027         if $archive eq '-';
6028
6029     my $info = PVE::Storage::archive_info($archive);
6030     my $format = $opts->{format} // $info->{format};
6031     my $comp = $info->{compression};
6032
6033     # try to detect archive format
6034     if ($format eq 'tar') {
6035         return restore_tar_archive($archive, $vmid, $user, $opts);
6036     } else {
6037         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6038     }
6039 }
6040
6041 # hepler to remove disks that will not be used after restore
6042 my $restore_cleanup_oldconf = sub {
6043     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6044
6045     PVE::QemuConfig->foreach_volume($oldconf, sub {
6046         my ($ds, $drive) = @_;
6047
6048         return if drive_is_cdrom($drive, 1);
6049
6050         my $volid = $drive->{file};
6051         return if !$volid || $volid =~ m|^/|;
6052
6053         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6054         return if !$path || !$owner || ($owner != $vmid);
6055
6056         # Note: only delete disk we want to restore
6057         # other volumes will become unused
6058         if ($virtdev_hash->{$ds}) {
6059             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6060             if (my $err = $@) {
6061                 warn $err;
6062             }
6063         }
6064     });
6065
6066     # delete vmstate files, after the restore we have no snapshots anymore
6067     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6068         my $snap = $oldconf->{snapshots}->{$snapname};
6069         if ($snap->{vmstate}) {
6070             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6071             if (my $err = $@) {
6072                 warn $err;
6073             }
6074         }
6075     }
6076 };
6077
6078 # Helper to parse vzdump backup device hints
6079 #
6080 # $rpcenv: Environment, used to ckeck storage permissions
6081 # $user: User ID, to check storage permissions
6082 # $storecfg: Storage configuration
6083 # $fh: the file handle for reading the configuration
6084 # $devinfo: should contain device sizes for all backu-up'ed devices
6085 # $options: backup options (pool, default storage)
6086 #
6087 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6088 my $parse_backup_hints = sub {
6089     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6090
6091     my $virtdev_hash = {};
6092
6093     while (defined(my $line = <$fh>)) {
6094         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6095             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6096             die "archive does not contain data for drive '$virtdev'\n"
6097                 if !$devinfo->{$devname};
6098
6099             if (defined($options->{storage})) {
6100                 $storeid = $options->{storage} || 'local';
6101             } elsif (!$storeid) {
6102                 $storeid = 'local';
6103             }
6104             $format = 'raw' if !$format;
6105             $devinfo->{$devname}->{devname} = $devname;
6106             $devinfo->{$devname}->{virtdev} = $virtdev;
6107             $devinfo->{$devname}->{format} = $format;
6108             $devinfo->{$devname}->{storeid} = $storeid;
6109
6110             # check permission on storage
6111             my $pool = $options->{pool}; # todo: do we need that?
6112             if ($user ne 'root@pam') {
6113                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6114             }
6115
6116             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6117         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6118             my $virtdev = $1;
6119             my $drive = parse_drive($virtdev, $2);
6120             if (drive_is_cloudinit($drive)) {
6121                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6122                 $storeid = $options->{storage} if defined ($options->{storage});
6123                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6124                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6125
6126                 $virtdev_hash->{$virtdev} = {
6127                     format => $format,
6128                     storeid => $storeid,
6129                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6130                     is_cloudinit => 1,
6131                 };
6132             }
6133         }
6134     }
6135
6136     return $virtdev_hash;
6137 };
6138
6139 # Helper to allocate and activate all volumes required for a restore
6140 #
6141 # $storecfg: Storage configuration
6142 # $virtdev_hash: as returned by parse_backup_hints()
6143 #
6144 # Returns: { $virtdev => $volid }
6145 my $restore_allocate_devices = sub {
6146     my ($storecfg, $virtdev_hash, $vmid) = @_;
6147
6148     my $map = {};
6149     foreach my $virtdev (sort keys %$virtdev_hash) {
6150         my $d = $virtdev_hash->{$virtdev};
6151         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6152         my $storeid = $d->{storeid};
6153         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6154
6155         # test if requested format is supported
6156         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6157         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6158         $d->{format} = $defFormat if !$supported;
6159
6160         my $name;
6161         if ($d->{is_cloudinit}) {
6162             $name = "vm-$vmid-cloudinit";
6163             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6164             if ($scfg->{path}) {
6165                 $name .= ".$d->{format}";
6166             }
6167         }
6168
6169         my $volid = PVE::Storage::vdisk_alloc(
6170             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6171
6172         print STDERR "new volume ID is '$volid'\n";
6173         $d->{volid} = $volid;
6174
6175         PVE::Storage::activate_volumes($storecfg, [$volid]);
6176
6177         $map->{$virtdev} = $volid;
6178     }
6179
6180     return $map;
6181 };
6182
6183 sub restore_update_config_line {
6184     my ($cookie, $map, $line, $unique) = @_;
6185
6186     return '' if $line =~ m/^\#qmdump\#/;
6187     return '' if $line =~ m/^\#vzdump\#/;
6188     return '' if $line =~ m/^lock:/;
6189     return '' if $line =~ m/^unused\d+:/;
6190     return '' if $line =~ m/^parent:/;
6191
6192     my $res = '';
6193
6194     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6195     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6196         # try to convert old 1.X settings
6197         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6198         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6199             my ($model, $macaddr) = split(/\=/, $devconfig);
6200             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6201             my $net = {
6202                 model => $model,
6203                 bridge => "vmbr$ind",
6204                 macaddr => $macaddr,
6205             };
6206             my $netstr = print_net($net);
6207
6208             $res .= "net$cookie->{netcount}: $netstr\n";
6209             $cookie->{netcount}++;
6210         }
6211     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6212         my ($id, $netstr) = ($1, $2);
6213         my $net = parse_net($netstr);
6214         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6215         $netstr = print_net($net);
6216         $res .= "$id: $netstr\n";
6217     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6218         my $virtdev = $1;
6219         my $value = $3;
6220         my $di = parse_drive($virtdev, $value);
6221         if (defined($di->{backup}) && !$di->{backup}) {
6222             $res .= "#$line";
6223         } elsif ($map->{$virtdev}) {
6224             delete $di->{format}; # format can change on restore
6225             $di->{file} = $map->{$virtdev};
6226             $value = print_drive($di);
6227             $res .= "$virtdev: $value\n";
6228         } else {
6229             $res .= $line;
6230         }
6231     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6232         my $vmgenid = $1;
6233         if ($vmgenid ne '0') {
6234             # always generate a new vmgenid if there was a valid one setup
6235             $vmgenid = generate_uuid();
6236         }
6237         $res .= "vmgenid: $vmgenid\n";
6238     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6239         my ($uuid, $uuid_str);
6240         UUID::generate($uuid);
6241         UUID::unparse($uuid, $uuid_str);
6242         my $smbios1 = parse_smbios1($2);
6243         $smbios1->{uuid} = $uuid_str;
6244         $res .= $1.print_smbios1($smbios1)."\n";
6245     } else {
6246         $res .= $line;
6247     }
6248
6249     return $res;
6250 }
6251
6252 my $restore_deactivate_volumes = sub {
6253     my ($storecfg, $devinfo) = @_;
6254
6255     my $vollist = [];
6256     foreach my $devname (keys %$devinfo) {
6257         my $volid = $devinfo->{$devname}->{volid};
6258         push @$vollist, $volid if $volid;
6259     }
6260
6261     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6262 };
6263
6264 my $restore_destroy_volumes = sub {
6265     my ($storecfg, $devinfo) = @_;
6266
6267     foreach my $devname (keys %$devinfo) {
6268         my $volid = $devinfo->{$devname}->{volid};
6269         next if !$volid;
6270         eval {
6271             if ($volid =~ m|^/|) {
6272                 unlink $volid || die 'unlink failed\n';
6273             } else {
6274                 PVE::Storage::vdisk_free($storecfg, $volid);
6275             }
6276             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6277         };
6278         print STDERR "unable to cleanup '$volid' - $@" if $@;
6279     }
6280 };
6281
6282 sub scan_volids {
6283     my ($cfg, $vmid) = @_;
6284
6285     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6286
6287     my $volid_hash = {};
6288     foreach my $storeid (keys %$info) {
6289         foreach my $item (@{$info->{$storeid}}) {
6290             next if !($item->{volid} && $item->{size});
6291             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6292             $volid_hash->{$item->{volid}} = $item;
6293         }
6294     }
6295
6296     return $volid_hash;
6297 }
6298
6299 sub update_disk_config {
6300     my ($vmid, $conf, $volid_hash) = @_;
6301
6302     my $changes;
6303     my $prefix = "VM $vmid";
6304
6305     # used and unused disks
6306     my $referenced = {};
6307
6308     # Note: it is allowed to define multiple storages with same path (alias), so
6309     # we need to check both 'volid' and real 'path' (two different volid can point
6310     # to the same path).
6311
6312     my $referencedpath = {};
6313
6314     # update size info
6315     PVE::QemuConfig->foreach_volume($conf, sub {
6316         my ($opt, $drive) = @_;
6317
6318         my $volid = $drive->{file};
6319         return if !$volid;
6320         my $volume = $volid_hash->{$volid};
6321
6322         # mark volid as "in-use" for next step
6323         $referenced->{$volid} = 1;
6324         if ($volume && (my $path = $volume->{path})) {
6325             $referencedpath->{$path} = 1;
6326         }
6327
6328         return if drive_is_cdrom($drive);
6329         return if !$volume;
6330
6331         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6332         if (defined($updated)) {
6333             $changes = 1;
6334             $conf->{$opt} = print_drive($updated);
6335             print "$prefix ($opt): $msg\n";
6336         }
6337     });
6338
6339     # remove 'unusedX' entry if volume is used
6340     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6341         my ($opt, $drive) = @_;
6342
6343         my $volid = $drive->{file};
6344         return if !$volid;
6345
6346         my $path;
6347         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6348         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6349             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6350             $changes = 1;
6351             delete $conf->{$opt};
6352         }
6353
6354         $referenced->{$volid} = 1;
6355         $referencedpath->{$path} = 1 if $path;
6356     });
6357
6358     foreach my $volid (sort keys %$volid_hash) {
6359         next if $volid =~ m/vm-$vmid-state-/;
6360         next if $referenced->{$volid};
6361         my $path = $volid_hash->{$volid}->{path};
6362         next if !$path; # just to be sure
6363         next if $referencedpath->{$path};
6364         $changes = 1;
6365         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6366         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6367         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6368     }
6369
6370     return $changes;
6371 }
6372
6373 sub rescan {
6374     my ($vmid, $nolock, $dryrun) = @_;
6375
6376     my $cfg = PVE::Storage::config();
6377
6378     print "rescan volumes...\n";
6379     my $volid_hash = scan_volids($cfg, $vmid);
6380
6381     my $updatefn =  sub {
6382         my ($vmid) = @_;
6383
6384         my $conf = PVE::QemuConfig->load_config($vmid);
6385
6386         PVE::QemuConfig->check_lock($conf);
6387
6388         my $vm_volids = {};
6389         foreach my $volid (keys %$volid_hash) {
6390             my $info = $volid_hash->{$volid};
6391             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6392         }
6393
6394         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6395
6396         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6397     };
6398
6399     if (defined($vmid)) {
6400         if ($nolock) {
6401             &$updatefn($vmid);
6402         } else {
6403             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6404         }
6405     } else {
6406         my $vmlist = config_list();
6407         foreach my $vmid (keys %$vmlist) {
6408             if ($nolock) {
6409                 &$updatefn($vmid);
6410             } else {
6411                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6412             }
6413         }
6414     }
6415 }
6416
6417 sub restore_proxmox_backup_archive {
6418     my ($archive, $vmid, $user, $options) = @_;
6419
6420     my $storecfg = PVE::Storage::config();
6421
6422     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6423     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6424
6425     my $fingerprint = $scfg->{fingerprint};
6426     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6427
6428     my $repo = PVE::PBSClient::get_repository($scfg);
6429
6430     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6431     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6432     local $ENV{PBS_PASSWORD} = $password;
6433     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6434
6435     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6436         PVE::Storage::parse_volname($storecfg, $archive);
6437
6438     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6439
6440     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6441
6442     my $tmpdir = "/var/tmp/vzdumptmp$$";
6443     rmtree $tmpdir;
6444     mkpath $tmpdir;
6445
6446     my $conffile = PVE::QemuConfig->config_file($vmid);
6447      # disable interrupts (always do cleanups)
6448     local $SIG{INT} =
6449         local $SIG{TERM} =
6450         local $SIG{QUIT} =
6451         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6452
6453     # Note: $oldconf is undef if VM does not exists
6454     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6455     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6456     my $new_conf_raw = '';
6457
6458     my $rpcenv = PVE::RPCEnvironment::get();
6459     my $devinfo = {};
6460
6461     eval {
6462         # enable interrupts
6463         local $SIG{INT} =
6464             local $SIG{TERM} =
6465             local $SIG{QUIT} =
6466             local $SIG{HUP} =
6467             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6468
6469         my $cfgfn = "$tmpdir/qemu-server.conf";
6470         my $firewall_config_fn = "$tmpdir/fw.conf";
6471         my $index_fn = "$tmpdir/index.json";
6472
6473         my $cmd = "restore";
6474
6475         my $param = [$pbs_backup_name, "index.json", $index_fn];
6476         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6477         my $index = PVE::Tools::file_get_contents($index_fn);
6478         $index = decode_json($index);
6479
6480         # print Dumper($index);
6481         foreach my $info (@{$index->{files}}) {
6482             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6483                 my $devname = $1;
6484                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6485                     $devinfo->{$devname}->{size} = $1;
6486                 } else {
6487                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6488                 }
6489             }
6490         }
6491
6492         my $is_qemu_server_backup = scalar(
6493             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6494         );
6495         if (!$is_qemu_server_backup) {
6496             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6497         }
6498         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6499
6500         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6501         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6502
6503         if ($has_firewall_config) {
6504             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6505             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6506
6507             my $pve_firewall_dir = '/etc/pve/firewall';
6508             mkdir $pve_firewall_dir; # make sure the dir exists
6509             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6510         }
6511
6512         my $fh = IO::File->new($cfgfn, "r") ||
6513             die "unable to read qemu-server.conf - $!\n";
6514
6515         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6516
6517         # fixme: rate limit?
6518
6519         # create empty/temp config
6520         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6521
6522         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6523
6524         # allocate volumes
6525         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6526
6527         foreach my $virtdev (sort keys %$virtdev_hash) {
6528             my $d = $virtdev_hash->{$virtdev};
6529             next if $d->{is_cloudinit}; # no need to restore cloudinit
6530
6531             # this fails if storage is unavailable
6532             my $volid = $d->{volid};
6533             my $path = PVE::Storage::path($storecfg, $volid);
6534
6535             # for live-restore we only want to preload the efidisk and TPM state
6536             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6537
6538             my $pbs_restore_cmd = [
6539                 '/usr/bin/pbs-restore',
6540                 '--repository', $repo,
6541                 $pbs_backup_name,
6542                 "$d->{devname}.img.fidx",
6543                 $path,
6544                 '--verbose',
6545                 ];
6546
6547             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6548             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6549
6550             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6551                 push @$pbs_restore_cmd, '--skip-zero';
6552             }
6553
6554             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6555             print "restore proxmox backup image: $dbg_cmdstring\n";
6556             run_command($pbs_restore_cmd);
6557         }
6558
6559         $fh->seek(0, 0) || die "seek failed - $!\n";
6560
6561         my $cookie = { netcount => 0 };
6562         while (defined(my $line = <$fh>)) {
6563             $new_conf_raw .= restore_update_config_line(
6564                 $cookie,
6565                 $map,
6566                 $line,
6567                 $options->{unique},
6568             );
6569         }
6570
6571         $fh->close();
6572     };
6573     my $err = $@;
6574
6575     if ($err || !$options->{live}) {
6576         $restore_deactivate_volumes->($storecfg, $devinfo);
6577     }
6578
6579     rmtree $tmpdir;
6580
6581     if ($err) {
6582         $restore_destroy_volumes->($storecfg, $devinfo);
6583         die $err;
6584     }
6585
6586     if ($options->{live}) {
6587         # keep lock during live-restore
6588         $new_conf_raw .= "\nlock: create";
6589     }
6590
6591     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6592
6593     PVE::Cluster::cfs_update(); # make sure we read new file
6594
6595     eval { rescan($vmid, 1); };
6596     warn $@ if $@;
6597
6598     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6599
6600     if ($options->{live}) {
6601         # enable interrupts
6602         local $SIG{INT} =
6603             local $SIG{TERM} =
6604             local $SIG{QUIT} =
6605             local $SIG{HUP} =
6606             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6607
6608         my $conf = PVE::QemuConfig->load_config($vmid);
6609         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6610
6611         # these special drives are already restored before start
6612         delete $devinfo->{'drive-efidisk0'};
6613         delete $devinfo->{'drive-tpmstate0-backup'};
6614         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6615
6616         PVE::QemuConfig->remove_lock($vmid, "create");
6617     }
6618 }
6619
6620 sub pbs_live_restore {
6621     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6622
6623     print "starting VM for live-restore\n";
6624     print "repository: '$repo', snapshot: '$snap'\n";
6625
6626     my $pbs_backing = {};
6627     for my $ds (keys %$restored_disks) {
6628         $ds =~ m/^drive-(.*)$/;
6629         my $confname = $1;
6630         $pbs_backing->{$confname} = {
6631             repository => $repo,
6632             snapshot => $snap,
6633             archive => "$ds.img.fidx",
6634         };
6635         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6636
6637         my $drive = parse_drive($confname, $conf->{$confname});
6638         print "restoring '$ds' to '$drive->{file}'\n";
6639     }
6640
6641     my $drives_streamed = 0;
6642     eval {
6643         # make sure HA doesn't interrupt our restore by stopping the VM
6644         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6645             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6646         }
6647
6648         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6649         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6650         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6651
6652         my $qmeventd_fd = register_qmeventd_handle($vmid);
6653
6654         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6655         # this will effectively collapse the backing image chain consisting of
6656         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6657         # removes itself once all backing images vanish with 'auto-remove=on')
6658         my $jobs = {};
6659         for my $ds (sort keys %$restored_disks) {
6660             my $job_id = "restore-$ds";
6661             mon_cmd($vmid, 'block-stream',
6662                 'job-id' => $job_id,
6663                 device => "$ds",
6664             );
6665             $jobs->{$job_id} = {};
6666         }
6667
6668         mon_cmd($vmid, 'cont');
6669         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6670
6671         print "restore-drive jobs finished successfully, removing all tracking block devices"
6672             ." to disconnect from Proxmox Backup Server\n";
6673
6674         for my $ds (sort keys %$restored_disks) {
6675             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6676         }
6677
6678         close($qmeventd_fd);
6679     };
6680
6681     my $err = $@;
6682
6683     if ($err) {
6684         warn "An error occured during live-restore: $err\n";
6685         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6686         die "live-restore failed\n";
6687     }
6688 }
6689
6690 sub restore_vma_archive {
6691     my ($archive, $vmid, $user, $opts, $comp) = @_;
6692
6693     my $readfrom = $archive;
6694
6695     my $cfg = PVE::Storage::config();
6696     my $commands = [];
6697     my $bwlimit = $opts->{bwlimit};
6698
6699     my $dbg_cmdstring = '';
6700     my $add_pipe = sub {
6701         my ($cmd) = @_;
6702         push @$commands, $cmd;
6703         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6704         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6705         $readfrom = '-';
6706     };
6707
6708     my $input = undef;
6709     if ($archive eq '-') {
6710         $input = '<&STDIN';
6711     } else {
6712         # If we use a backup from a PVE defined storage we also consider that
6713         # storage's rate limit:
6714         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6715         if (defined($volid)) {
6716             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6717             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6718             if ($readlimit) {
6719                 print STDERR "applying read rate limit: $readlimit\n";
6720                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6721                 $add_pipe->($cstream);
6722             }
6723         }
6724     }
6725
6726     if ($comp) {
6727         my $info = PVE::Storage::decompressor_info('vma', $comp);
6728         my $cmd = $info->{decompressor};
6729         push @$cmd, $readfrom;
6730         $add_pipe->($cmd);
6731     }
6732
6733     my $tmpdir = "/var/tmp/vzdumptmp$$";
6734     rmtree $tmpdir;
6735
6736     # disable interrupts (always do cleanups)
6737     local $SIG{INT} =
6738         local $SIG{TERM} =
6739         local $SIG{QUIT} =
6740         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6741
6742     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6743     POSIX::mkfifo($mapfifo, 0600);
6744     my $fifofh;
6745     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6746
6747     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6748
6749     my $oldtimeout;
6750     my $timeout = 5;
6751
6752     my $devinfo = {};
6753
6754     my $rpcenv = PVE::RPCEnvironment::get();
6755
6756     my $conffile = PVE::QemuConfig->config_file($vmid);
6757
6758     # Note: $oldconf is undef if VM does not exist
6759     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6760     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6761     my $new_conf_raw = '';
6762
6763     my %storage_limits;
6764
6765     my $print_devmap = sub {
6766         my $cfgfn = "$tmpdir/qemu-server.conf";
6767
6768         # we can read the config - that is already extracted
6769         my $fh = IO::File->new($cfgfn, "r") ||
6770             die "unable to read qemu-server.conf - $!\n";
6771
6772         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6773         if (-f $fwcfgfn) {
6774             my $pve_firewall_dir = '/etc/pve/firewall';
6775             mkdir $pve_firewall_dir; # make sure the dir exists
6776             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6777         }
6778
6779         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6780
6781         foreach my $info (values %{$virtdev_hash}) {
6782             my $storeid = $info->{storeid};
6783             next if defined($storage_limits{$storeid});
6784
6785             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6786             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6787             $storage_limits{$storeid} = $limit * 1024;
6788         }
6789
6790         foreach my $devname (keys %$devinfo) {
6791             die "found no device mapping information for device '$devname'\n"
6792                 if !$devinfo->{$devname}->{virtdev};
6793         }
6794
6795         # create empty/temp config
6796         if ($oldconf) {
6797             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6798             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6799         }
6800
6801         # allocate volumes
6802         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6803
6804         # print restore information to $fifofh
6805         foreach my $virtdev (sort keys %$virtdev_hash) {
6806             my $d = $virtdev_hash->{$virtdev};
6807             next if $d->{is_cloudinit}; # no need to restore cloudinit
6808
6809             my $storeid = $d->{storeid};
6810             my $volid = $d->{volid};
6811
6812             my $map_opts = '';
6813             if (my $limit = $storage_limits{$storeid}) {
6814                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6815             }
6816
6817             my $write_zeros = 1;
6818             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6819                 $write_zeros = 0;
6820             }
6821
6822             my $path = PVE::Storage::path($cfg, $volid);
6823
6824             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6825
6826             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6827         }
6828
6829         $fh->seek(0, 0) || die "seek failed - $!\n";
6830
6831         my $cookie = { netcount => 0 };
6832         while (defined(my $line = <$fh>)) {
6833             $new_conf_raw .= restore_update_config_line(
6834                 $cookie,
6835                 $map,
6836                 $line,
6837                 $opts->{unique},
6838             );
6839         }
6840
6841         $fh->close();
6842     };
6843
6844     eval {
6845         # enable interrupts
6846         local $SIG{INT} =
6847             local $SIG{TERM} =
6848             local $SIG{QUIT} =
6849             local $SIG{HUP} =
6850             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6851         local $SIG{ALRM} = sub { die "got timeout\n"; };
6852
6853         $oldtimeout = alarm($timeout);
6854
6855         my $parser = sub {
6856             my $line = shift;
6857
6858             print "$line\n";
6859
6860             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6861                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6862                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6863             } elsif ($line =~ m/^CTIME: /) {
6864                 # we correctly received the vma config, so we can disable
6865                 # the timeout now for disk allocation (set to 10 minutes, so
6866                 # that we always timeout if something goes wrong)
6867                 alarm(600);
6868                 &$print_devmap();
6869                 print $fifofh "done\n";
6870                 my $tmp = $oldtimeout || 0;
6871                 $oldtimeout = undef;
6872                 alarm($tmp);
6873                 close($fifofh);
6874                 $fifofh = undef;
6875             }
6876         };
6877
6878         print "restore vma archive: $dbg_cmdstring\n";
6879         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6880     };
6881     my $err = $@;
6882
6883     alarm($oldtimeout) if $oldtimeout;
6884
6885     $restore_deactivate_volumes->($cfg, $devinfo);
6886
6887     close($fifofh) if $fifofh;
6888     unlink $mapfifo;
6889     rmtree $tmpdir;
6890
6891     if ($err) {
6892         $restore_destroy_volumes->($cfg, $devinfo);
6893         die $err;
6894     }
6895
6896     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6897
6898     PVE::Cluster::cfs_update(); # make sure we read new file
6899
6900     eval { rescan($vmid, 1); };
6901     warn $@ if $@;
6902
6903     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6904 }
6905
6906 sub restore_tar_archive {
6907     my ($archive, $vmid, $user, $opts) = @_;
6908
6909     if ($archive ne '-') {
6910         my $firstfile = tar_archive_read_firstfile($archive);
6911         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6912             if $firstfile ne 'qemu-server.conf';
6913     }
6914
6915     my $storecfg = PVE::Storage::config();
6916
6917     # avoid zombie disks when restoring over an existing VM -> cleanup first
6918     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6919     # skiplock=1 because qmrestore has set the 'create' lock itself already
6920     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6921     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6922
6923     my $tocmd = "/usr/lib/qemu-server/qmextract";
6924
6925     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6926     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6927     $tocmd .= ' --prealloc' if $opts->{prealloc};
6928     $tocmd .= ' --info' if $opts->{info};
6929
6930     # tar option "xf" does not autodetect compression when read from STDIN,
6931     # so we pipe to zcat
6932     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6933         PVE::Tools::shellquote("--to-command=$tocmd");
6934
6935     my $tmpdir = "/var/tmp/vzdumptmp$$";
6936     mkpath $tmpdir;
6937
6938     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6939     local $ENV{VZDUMP_VMID} = $vmid;
6940     local $ENV{VZDUMP_USER} = $user;
6941
6942     my $conffile = PVE::QemuConfig->config_file($vmid);
6943     my $new_conf_raw = '';
6944
6945     # disable interrupts (always do cleanups)
6946     local $SIG{INT} =
6947         local $SIG{TERM} =
6948         local $SIG{QUIT} =
6949         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6950
6951     eval {
6952         # enable interrupts
6953         local $SIG{INT} =
6954             local $SIG{TERM} =
6955             local $SIG{QUIT} =
6956             local $SIG{HUP} =
6957             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6958
6959         if ($archive eq '-') {
6960             print "extracting archive from STDIN\n";
6961             run_command($cmd, input => "<&STDIN");
6962         } else {
6963             print "extracting archive '$archive'\n";
6964             run_command($cmd);
6965         }
6966
6967         return if $opts->{info};
6968
6969         # read new mapping
6970         my $map = {};
6971         my $statfile = "$tmpdir/qmrestore.stat";
6972         if (my $fd = IO::File->new($statfile, "r")) {
6973             while (defined (my $line = <$fd>)) {
6974                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6975                     $map->{$1} = $2 if $1;
6976                 } else {
6977                     print STDERR "unable to parse line in statfile - $line\n";
6978                 }
6979             }
6980             $fd->close();
6981         }
6982
6983         my $confsrc = "$tmpdir/qemu-server.conf";
6984
6985         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6986
6987         my $cookie = { netcount => 0 };
6988         while (defined (my $line = <$srcfd>)) {
6989             $new_conf_raw .= restore_update_config_line(
6990                 $cookie,
6991                 $map,
6992                 $line,
6993                 $opts->{unique},
6994             );
6995         }
6996
6997         $srcfd->close();
6998     };
6999     if (my $err = $@) {
7000         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7001         die $err;
7002     }
7003
7004     rmtree $tmpdir;
7005
7006     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7007
7008     PVE::Cluster::cfs_update(); # make sure we read new file
7009
7010     eval { rescan($vmid, 1); };
7011     warn $@ if $@;
7012 };
7013
7014 sub foreach_storage_used_by_vm {
7015     my ($conf, $func) = @_;
7016
7017     my $sidhash = {};
7018
7019     PVE::QemuConfig->foreach_volume($conf, sub {
7020         my ($ds, $drive) = @_;
7021         return if drive_is_cdrom($drive);
7022
7023         my $volid = $drive->{file};
7024
7025         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7026         $sidhash->{$sid} = $sid if $sid;
7027     });
7028
7029     foreach my $sid (sort keys %$sidhash) {
7030         &$func($sid);
7031     }
7032 }
7033
7034 my $qemu_snap_storage = {
7035     rbd => 1,
7036 };
7037 sub do_snapshots_with_qemu {
7038     my ($storecfg, $volid, $deviceid) = @_;
7039
7040     return if $deviceid =~ m/tpmstate0/;
7041
7042     my $storage_name = PVE::Storage::parse_volume_id($volid);
7043     my $scfg = $storecfg->{ids}->{$storage_name};
7044     die "could not find storage '$storage_name'\n" if !defined($scfg);
7045
7046     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7047         return 1;
7048     }
7049
7050     if ($volid =~ m/\.(qcow2|qed)$/){
7051         return 1;
7052     }
7053
7054     return;
7055 }
7056
7057 sub qga_check_running {
7058     my ($vmid, $nowarn) = @_;
7059
7060     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7061     if ($@) {
7062         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7063         return 0;
7064     }
7065     return 1;
7066 }
7067
7068 sub template_create {
7069     my ($vmid, $conf, $disk) = @_;
7070
7071     my $storecfg = PVE::Storage::config();
7072
7073     PVE::QemuConfig->foreach_volume($conf, sub {
7074         my ($ds, $drive) = @_;
7075
7076         return if drive_is_cdrom($drive);
7077         return if $disk && $ds ne $disk;
7078
7079         my $volid = $drive->{file};
7080         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7081
7082         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7083         $drive->{file} = $voliddst;
7084         $conf->{$ds} = print_drive($drive);
7085         PVE::QemuConfig->write_config($vmid, $conf);
7086     });
7087 }
7088
7089 sub convert_iscsi_path {
7090     my ($path) = @_;
7091
7092     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7093         my $portal = $1;
7094         my $target = $2;
7095         my $lun = $3;
7096
7097         my $initiator_name = get_initiator_name();
7098
7099         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7100                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7101     }
7102
7103     die "cannot convert iscsi path '$path', unkown format\n";
7104 }
7105
7106 sub qemu_img_convert {
7107     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7108
7109     my $storecfg = PVE::Storage::config();
7110     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7111     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7112
7113     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7114
7115     my $cachemode;
7116     my $src_path;
7117     my $src_is_iscsi = 0;
7118     my $src_format;
7119
7120     if ($src_storeid) {
7121         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7122         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7123         $src_format = qemu_img_format($src_scfg, $src_volname);
7124         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7125         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7126         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7127     } elsif (-f $src_volid) {
7128         $src_path = $src_volid;
7129         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7130             $src_format = $1;
7131         }
7132     }
7133
7134     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7135
7136     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7137     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7138     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7139     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7140
7141     my $cmd = [];
7142     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7143     push @$cmd, '-l', "snapshot.name=$snapname"
7144         if $snapname && $src_format && $src_format eq "qcow2";
7145     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7146     push @$cmd, '-T', $cachemode if defined($cachemode);
7147
7148     if ($src_is_iscsi) {
7149         push @$cmd, '--image-opts';
7150         $src_path = convert_iscsi_path($src_path);
7151     } elsif ($src_format) {
7152         push @$cmd, '-f', $src_format;
7153     }
7154
7155     if ($dst_is_iscsi) {
7156         push @$cmd, '--target-image-opts';
7157         $dst_path = convert_iscsi_path($dst_path);
7158     } else {
7159         push @$cmd, '-O', $dst_format;
7160     }
7161
7162     push @$cmd, $src_path;
7163
7164     if (!$dst_is_iscsi && $is_zero_initialized) {
7165         push @$cmd, "zeroinit:$dst_path";
7166     } else {
7167         push @$cmd, $dst_path;
7168     }
7169
7170     my $parser = sub {
7171         my $line = shift;
7172         if($line =~ m/\((\S+)\/100\%\)/){
7173             my $percent = $1;
7174             my $transferred = int($size * $percent / 100);
7175             my $total_h = render_bytes($size, 1);
7176             my $transferred_h = render_bytes($transferred, 1);
7177
7178             print "transferred $transferred_h of $total_h ($percent%)\n";
7179         }
7180
7181     };
7182
7183     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7184     my $err = $@;
7185     die "copy failed: $err" if $err;
7186 }
7187
7188 sub qemu_img_format {
7189     my ($scfg, $volname) = @_;
7190
7191     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7192         return $1;
7193     } else {
7194         return "raw";
7195     }
7196 }
7197
7198 sub qemu_drive_mirror {
7199     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7200
7201     $jobs = {} if !$jobs;
7202
7203     my $qemu_target;
7204     my $format;
7205     $jobs->{"drive-$drive"} = {};
7206
7207     if ($dst_volid =~ /^nbd:/) {
7208         $qemu_target = $dst_volid;
7209         $format = "nbd";
7210     } else {
7211         my $storecfg = PVE::Storage::config();
7212         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7213
7214         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7215
7216         $format = qemu_img_format($dst_scfg, $dst_volname);
7217
7218         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7219
7220         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7221     }
7222
7223     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7224     $opts->{format} = $format if $format;
7225
7226     if (defined($src_bitmap)) {
7227         $opts->{sync} = 'incremental';
7228         $opts->{bitmap} = $src_bitmap;
7229         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7230     }
7231
7232     if (defined($bwlimit)) {
7233         $opts->{speed} = $bwlimit * 1024;
7234         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7235     } else {
7236         print "drive mirror is starting for drive-$drive\n";
7237     }
7238
7239     # if a job already runs for this device we get an error, catch it for cleanup
7240     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7241     if (my $err = $@) {
7242         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7243         warn "$@\n" if $@;
7244         die "mirroring error: $err\n";
7245     }
7246
7247     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7248 }
7249
7250 # $completion can be either
7251 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7252 # 'cancel': wait until all jobs are ready, block-job-cancel them
7253 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7254 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7255 sub qemu_drive_mirror_monitor {
7256     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7257
7258     $completion //= 'complete';
7259     $op //= "mirror";
7260
7261     eval {
7262         my $err_complete = 0;
7263
7264         my $starttime = time ();
7265         while (1) {
7266             die "block job ('$op') timed out\n" if $err_complete > 300;
7267
7268             my $stats = mon_cmd($vmid, "query-block-jobs");
7269             my $ctime = time();
7270
7271             my $running_jobs = {};
7272             for my $stat (@$stats) {
7273                 next if $stat->{type} ne $op;
7274                 $running_jobs->{$stat->{device}} = $stat;
7275             }
7276
7277             my $readycounter = 0;
7278
7279             for my $job_id (sort keys %$jobs) {
7280                 my $job = $running_jobs->{$job_id};
7281
7282                 my $vanished = !defined($job);
7283                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7284                 if($complete || ($vanished && $completion eq 'auto')) {
7285                     print "$job_id: $op-job finished\n";
7286                     delete $jobs->{$job_id};
7287                     next;
7288                 }
7289
7290                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7291
7292                 my $busy = $job->{busy};
7293                 my $ready = $job->{ready};
7294                 if (my $total = $job->{len}) {
7295                     my $transferred = $job->{offset} || 0;
7296                     my $remaining = $total - $transferred;
7297                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7298
7299                     my $duration = $ctime - $starttime;
7300                     my $total_h = render_bytes($total, 1);
7301                     my $transferred_h = render_bytes($transferred, 1);
7302
7303                     my $status = sprintf(
7304                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7305                         render_duration($duration),
7306                     );
7307
7308                     if ($ready) {
7309                         if ($busy) {
7310                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7311                         } else {
7312                             $status .= ", ready";
7313                         }
7314                     }
7315                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7316                     $jobs->{$job_id}->{ready} = $ready;
7317                 }
7318
7319                 $readycounter++ if $job->{ready};
7320             }
7321
7322             last if scalar(keys %$jobs) == 0;
7323
7324             if ($readycounter == scalar(keys %$jobs)) {
7325                 print "all '$op' jobs are ready\n";
7326
7327                 # do the complete later (or has already been done)
7328                 last if $completion eq 'skip' || $completion eq 'auto';
7329
7330                 if ($vmiddst && $vmiddst != $vmid) {
7331                     my $agent_running = $qga && qga_check_running($vmid);
7332                     if ($agent_running) {
7333                         print "freeze filesystem\n";
7334                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7335                     } else {
7336                         print "suspend vm\n";
7337                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7338                     }
7339
7340                     # if we clone a disk for a new target vm, we don't switch the disk
7341                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7342
7343                     if ($agent_running) {
7344                         print "unfreeze filesystem\n";
7345                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7346                     } else {
7347                         print "resume vm\n";
7348                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7349                     }
7350
7351                     last;
7352                 } else {
7353
7354                     for my $job_id (sort keys %$jobs) {
7355                         # try to switch the disk if source and destination are on the same guest
7356                         print "$job_id: Completing block job_id...\n";
7357
7358                         my $op;
7359                         if ($completion eq 'complete') {
7360                             $op = 'block-job-complete';
7361                         } elsif ($completion eq 'cancel') {
7362                             $op = 'block-job-cancel';
7363                         } else {
7364                             die "invalid completion value: $completion\n";
7365                         }
7366                         eval { mon_cmd($vmid, $op, device => $job_id) };
7367                         if ($@ =~ m/cannot be completed/) {
7368                             print "$job_id: block job cannot be completed, trying again.\n";
7369                             $err_complete++;
7370                         }else {
7371                             print "$job_id: Completed successfully.\n";
7372                             $jobs->{$job_id}->{complete} = 1;
7373                         }
7374                     }
7375                 }
7376             }
7377             sleep 1;
7378         }
7379     };
7380     my $err = $@;
7381
7382     if ($err) {
7383         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7384         die "block job ($op) error: $err";
7385     }
7386 }
7387
7388 sub qemu_blockjobs_cancel {
7389     my ($vmid, $jobs) = @_;
7390
7391     foreach my $job (keys %$jobs) {
7392         print "$job: Cancelling block job\n";
7393         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7394         $jobs->{$job}->{cancel} = 1;
7395     }
7396
7397     while (1) {
7398         my $stats = mon_cmd($vmid, "query-block-jobs");
7399
7400         my $running_jobs = {};
7401         foreach my $stat (@$stats) {
7402             $running_jobs->{$stat->{device}} = $stat;
7403         }
7404
7405         foreach my $job (keys %$jobs) {
7406
7407             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7408                 print "$job: Done.\n";
7409                 delete $jobs->{$job};
7410             }
7411         }
7412
7413         last if scalar(keys %$jobs) == 0;
7414
7415         sleep 1;
7416     }
7417 }
7418
7419 sub clone_disk {
7420     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7421         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7422
7423     my $newvolid;
7424
7425     if (!$full) {
7426         print "create linked clone of drive $drivename ($drive->{file})\n";
7427         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7428         push @$newvollist, $newvolid;
7429     } else {
7430
7431         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7432         $storeid = $storage if $storage;
7433
7434         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7435
7436         print "create full clone of drive $drivename ($drive->{file})\n";
7437         my $name = undef;
7438         my $size = undef;
7439         if (drive_is_cloudinit($drive)) {
7440             $name = "vm-$newvmid-cloudinit";
7441             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7442             if ($scfg->{path}) {
7443                 $name .= ".$dst_format";
7444             }
7445             $snapname = undef;
7446             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7447         } elsif ($drivename eq 'efidisk0') {
7448             $size = get_efivars_size($conf);
7449         } elsif ($drivename eq 'tpmstate0') {
7450             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7451         } else {
7452             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7453         }
7454         $newvolid = PVE::Storage::vdisk_alloc(
7455             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7456         );
7457         push @$newvollist, $newvolid;
7458
7459         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7460
7461         if (drive_is_cloudinit($drive)) {
7462             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7463             # if this is the case, we have to complete any block-jobs still there from
7464             # previous drive-mirrors
7465             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7466                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7467             }
7468             goto no_data_clone;
7469         }
7470
7471         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7472         if (!$running || $snapname) {
7473             # TODO: handle bwlimits
7474             if ($drivename eq 'efidisk0') {
7475                 # the relevant data on the efidisk may be smaller than the source
7476                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7477                 # that is given by the OVMF_VARS.fd
7478                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7479                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7480
7481                 # better for Ceph if block size is not too small, see bug #3324
7482                 my $bs = 1024*1024;
7483
7484                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7485                     "if=$src_path", "of=$dst_path"]);
7486             } else {
7487                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7488             }
7489         } else {
7490
7491             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7492
7493             my $kvmver = get_running_qemu_version ($vmid);
7494             if (!min_version($kvmver, 2, 7)) {
7495                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7496                     if $drive->{iothread};
7497             }
7498
7499             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7500                 $completion, $qga, $bwlimit);
7501         }
7502     }
7503
7504 no_data_clone:
7505     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7506
7507     my $disk = $drive;
7508     $disk->{format} = undef;
7509     $disk->{file} = $newvolid;
7510     $disk->{size} = $size if defined($size);
7511
7512     return $disk;
7513 }
7514
7515 sub get_running_qemu_version {
7516     my ($vmid) = @_;
7517     my $res = mon_cmd($vmid, "query-version");
7518     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7519 }
7520
7521 sub qemu_use_old_bios_files {
7522     my ($machine_type) = @_;
7523
7524     return if !$machine_type;
7525
7526     my $use_old_bios_files = undef;
7527
7528     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7529         $machine_type = $1;
7530         $use_old_bios_files = 1;
7531     } else {
7532         my $version = extract_version($machine_type, kvm_user_version());
7533         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7534         # load new efi bios files on migration. So this hack is required to allow
7535         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7536         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7537         $use_old_bios_files = !min_version($version, 2, 4);
7538     }
7539
7540     return ($use_old_bios_files, $machine_type);
7541 }
7542
7543 sub get_efivars_size {
7544     my ($conf) = @_;
7545     my $arch = get_vm_arch($conf);
7546     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7547     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7548     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7549     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7550     return -s $ovmf_vars;
7551 }
7552
7553 sub update_efidisk_size {
7554     my ($conf) = @_;
7555
7556     return if !defined($conf->{efidisk0});
7557
7558     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7559     $disk->{size} = get_efivars_size($conf);
7560     $conf->{efidisk0} = print_drive($disk);
7561
7562     return;
7563 }
7564
7565 sub update_tpmstate_size {
7566     my ($conf) = @_;
7567
7568     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7569     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7570     $conf->{tpmstate0} = print_drive($disk);
7571 }
7572
7573 sub create_efidisk($$$$$$$) {
7574     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7575
7576     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7577     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7578
7579     my $vars_size_b = -s $ovmf_vars;
7580     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7581     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7582     PVE::Storage::activate_volumes($storecfg, [$volid]);
7583
7584     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7585     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7586
7587     return ($volid, $size/1024);
7588 }
7589
7590 sub vm_iothreads_list {
7591     my ($vmid) = @_;
7592
7593     my $res = mon_cmd($vmid, 'query-iothreads');
7594
7595     my $iothreads = {};
7596     foreach my $iothread (@$res) {
7597         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7598     }
7599
7600     return $iothreads;
7601 }
7602
7603 sub scsihw_infos {
7604     my ($conf, $drive) = @_;
7605
7606     my $maxdev = 0;
7607
7608     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7609         $maxdev = 7;
7610     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7611         $maxdev = 1;
7612     } else {
7613         $maxdev = 256;
7614     }
7615
7616     my $controller = int($drive->{index} / $maxdev);
7617     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7618         ? "virtioscsi"
7619         : "scsihw";
7620
7621     return ($maxdev, $controller, $controller_prefix);
7622 }
7623
7624 sub windows_version {
7625     my ($ostype) = @_;
7626
7627     return 0 if !$ostype;
7628
7629     my $winversion = 0;
7630
7631     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7632         $winversion = 5;
7633     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7634         $winversion = 6;
7635     } elsif ($ostype =~ m/^win(\d+)$/) {
7636         $winversion = $1;
7637     }
7638
7639     return $winversion;
7640 }
7641
7642 sub resolve_dst_disk_format {
7643         my ($storecfg, $storeid, $src_volname, $format) = @_;
7644         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7645
7646         if (!$format) {
7647             # if no target format is specified, use the source disk format as hint
7648             if ($src_volname) {
7649                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7650                 $format = qemu_img_format($scfg, $src_volname);
7651             } else {
7652                 return $defFormat;
7653             }
7654         }
7655
7656         # test if requested format is supported - else use default
7657         my $supported = grep { $_ eq $format } @$validFormats;
7658         $format = $defFormat if !$supported;
7659         return $format;
7660 }
7661
7662 # NOTE: if this logic changes, please update docs & possibly gui logic
7663 sub find_vmstate_storage {
7664     my ($conf, $storecfg) = @_;
7665
7666     # first, return storage from conf if set
7667     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7668
7669     my ($target, $shared, $local);
7670
7671     foreach_storage_used_by_vm($conf, sub {
7672         my ($sid) = @_;
7673         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7674         my $dst = $scfg->{shared} ? \$shared : \$local;
7675         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7676     });
7677
7678     # second, use shared storage where VM has at least one disk
7679     # third, use local storage where VM has at least one disk
7680     # fall back to local storage
7681     $target = $shared // $local // 'local';
7682
7683     return $target;
7684 }
7685
7686 sub generate_uuid {
7687     my ($uuid, $uuid_str);
7688     UUID::generate($uuid);
7689     UUID::unparse($uuid, $uuid_str);
7690     return $uuid_str;
7691 }
7692
7693 sub generate_smbios1_uuid {
7694     return "uuid=".generate_uuid();
7695 }
7696
7697 sub nbd_stop {
7698     my ($vmid) = @_;
7699
7700     mon_cmd($vmid, 'nbd-server-stop');
7701 }
7702
7703 sub create_reboot_request {
7704     my ($vmid) = @_;
7705     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7706         or die "failed to create reboot trigger file: $!\n";
7707     close($fh);
7708 }
7709
7710 sub clear_reboot_request {
7711     my ($vmid) = @_;
7712     my $path = "/run/qemu-server/$vmid.reboot";
7713     my $res = 0;
7714
7715     $res = unlink($path);
7716     die "could not remove reboot request for $vmid: $!"
7717         if !$res && $! != POSIX::ENOENT;
7718
7719     return $res;
7720 }
7721
7722 sub bootorder_from_legacy {
7723     my ($conf, $bootcfg) = @_;
7724
7725     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7726     my $bootindex_hash = {};
7727     my $i = 1;
7728     foreach my $o (split(//, $boot)) {
7729         $bootindex_hash->{$o} = $i*100;
7730         $i++;
7731     }
7732
7733     my $bootorder = {};
7734
7735     PVE::QemuConfig->foreach_volume($conf, sub {
7736         my ($ds, $drive) = @_;
7737
7738         if (drive_is_cdrom ($drive, 1)) {
7739             if ($bootindex_hash->{d}) {
7740                 $bootorder->{$ds} = $bootindex_hash->{d};
7741                 $bootindex_hash->{d} += 1;
7742             }
7743         } elsif ($bootindex_hash->{c}) {
7744             $bootorder->{$ds} = $bootindex_hash->{c}
7745                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7746             $bootindex_hash->{c} += 1;
7747         }
7748     });
7749
7750     if ($bootindex_hash->{n}) {
7751         for (my $i = 0; $i < $MAX_NETS; $i++) {
7752             my $netname = "net$i";
7753             next if !$conf->{$netname};
7754             $bootorder->{$netname} = $bootindex_hash->{n};
7755             $bootindex_hash->{n} += 1;
7756         }
7757     }
7758
7759     return $bootorder;
7760 }
7761
7762 # Generate default device list for 'boot: order=' property. Matches legacy
7763 # default boot order, but with explicit device names. This is important, since
7764 # the fallback for when neither 'order' nor the old format is specified relies
7765 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7766 sub get_default_bootdevices {
7767     my ($conf) = @_;
7768
7769     my @ret = ();
7770
7771     # harddisk
7772     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7773     push @ret, $first if $first;
7774
7775     # cdrom
7776     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7777     push @ret, $first if $first;
7778
7779     # network
7780     for (my $i = 0; $i < $MAX_NETS; $i++) {
7781         my $netname = "net$i";
7782         next if !$conf->{$netname};
7783         push @ret, $netname;
7784         last;
7785     }
7786
7787     return \@ret;
7788 }
7789
7790 sub device_bootorder {
7791     my ($conf) = @_;
7792
7793     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7794
7795     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7796
7797     my $bootorder = {};
7798     if (!defined($boot) || $boot->{legacy}) {
7799         $bootorder = bootorder_from_legacy($conf, $boot);
7800     } elsif ($boot->{order}) {
7801         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7802         for my $dev (PVE::Tools::split_list($boot->{order})) {
7803             $bootorder->{$dev} = $i++;
7804         }
7805     }
7806
7807     return $bootorder;
7808 }
7809
7810 sub register_qmeventd_handle {
7811     my ($vmid) = @_;
7812
7813     my $fh;
7814     my $peer = "/var/run/qmeventd.sock";
7815     my $count = 0;
7816
7817     for (;;) {
7818         $count++;
7819         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7820         last if $fh;
7821         if ($! != EINTR && $! != EAGAIN) {
7822             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7823         }
7824         if ($count > 4) {
7825             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7826               . "after $count retries\n";
7827         }
7828         usleep(25000);
7829     }
7830
7831     # send handshake to mark VM as backing up
7832     print $fh to_json({vzdump => {vmid => "$vmid"}});
7833
7834     # return handle to be closed later when inhibit is no longer required
7835     return $fh;
7836 }
7837
7838 # bash completion helper
7839
7840 sub complete_backup_archives {
7841     my ($cmdname, $pname, $cvalue) = @_;
7842
7843     my $cfg = PVE::Storage::config();
7844
7845     my $storeid;
7846
7847     if ($cvalue =~ m/^([^:]+):/) {
7848         $storeid = $1;
7849     }
7850
7851     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7852
7853     my $res = [];
7854     foreach my $id (keys %$data) {
7855         foreach my $item (@{$data->{$id}}) {
7856             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7857             push @$res, $item->{volid} if defined($item->{volid});
7858         }
7859     }
7860
7861     return $res;
7862 }
7863
7864 my $complete_vmid_full = sub {
7865     my ($running) = @_;
7866
7867     my $idlist = vmstatus();
7868
7869     my $res = [];
7870
7871     foreach my $id (keys %$idlist) {
7872         my $d = $idlist->{$id};
7873         if (defined($running)) {
7874             next if $d->{template};
7875             next if $running && $d->{status} ne 'running';
7876             next if !$running && $d->{status} eq 'running';
7877         }
7878         push @$res, $id;
7879
7880     }
7881     return $res;
7882 };
7883
7884 sub complete_vmid {
7885     return &$complete_vmid_full();
7886 }
7887
7888 sub complete_vmid_stopped {
7889     return &$complete_vmid_full(0);
7890 }
7891
7892 sub complete_vmid_running {
7893     return &$complete_vmid_full(1);
7894 }
7895
7896 sub complete_storage {
7897
7898     my $cfg = PVE::Storage::config();
7899     my $ids = $cfg->{ids};
7900
7901     my $res = [];
7902     foreach my $sid (keys %$ids) {
7903         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7904         next if !$ids->{$sid}->{content}->{images};
7905         push @$res, $sid;
7906     }
7907
7908     return $res;
7909 }
7910
7911 sub complete_migration_storage {
7912     my ($cmd, $param, $current_value, $all_args) = @_;
7913
7914     my $targetnode = @$all_args[1];
7915
7916     my $cfg = PVE::Storage::config();
7917     my $ids = $cfg->{ids};
7918
7919     my $res = [];
7920     foreach my $sid (keys %$ids) {
7921         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7922         next if !$ids->{$sid}->{content}->{images};
7923         push @$res, $sid;
7924     }
7925
7926     return $res;
7927 }
7928
7929 sub vm_is_paused {
7930     my ($vmid) = @_;
7931     my $qmpstatus = eval {
7932         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7933         mon_cmd($vmid, "query-status");
7934     };
7935     warn "$@\n" if $@;
7936     return $qmpstatus && $qmpstatus->{status} eq "paused";
7937 }
7938
7939 sub check_volume_storage_type {
7940     my ($storecfg, $vol) = @_;
7941
7942     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7943     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7944     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7945
7946     die "storage '$storeid' does not support content-type '$vtype'\n"
7947         if !$scfg->{content}->{$vtype};
7948
7949     return 1;
7950 }
7951
7952 1;