PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         default => [
  76             "$EDK2_FW_BASE/OVMF_CODE.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS.fd",
  78         ],
  79     },
  80     aarch64 => {
  81         default => [
  82             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  83             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  84         ],
  85     },
  86 };
  87
  88 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  89
  90 # Note about locking: we use flock on the config file protect against concurent actions.
  91 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
  92 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
  93 # But you can ignore this kind of lock with the --skiplock flag.
  94
  95 cfs_register_file('/qemu-server/',
  96                   \&parse_vm_config,
  97                   \&write_vm_config);
  98
  99 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 100     description => "Some command save/restore state from this location.",
 101     type => 'string',
 102     maxLength => 128,
 103     optional => 1,
 104 });
 105
 106 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 107         description => "Specifies the Qemu machine type.",
 108         type => 'string',
 109         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 110         maxLength => 40,
 111         optional => 1,
 112 });
 113
 114
 115 sub map_storage {
 116     my ($map, $source) = @_;
 117
 118     return $source if !defined($map);
 119
 120     return $map->{entries}->{$source}
 121         if $map->{entries} && defined($map->{entries}->{$source});
 122
 123     return $map->{default} if $map->{default};
 124
 125     # identity (fallback)
 126     return $source;
 127 }
 128
 129 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 130     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 131     type => 'string',
 132     format => 'storagepair-list',
 133     optional => 1,
 134 });
 135
 136 #no warnings 'redefine';
 137
 138 my $nodename_cache;
 139 sub nodename {
 140     $nodename_cache //= PVE::INotify::nodename();
 141     return $nodename_cache;
 142 }
 143
 144 my $watchdog_fmt = {
 145     model => {
 146         default_key => 1,
 147         type => 'string',
 148         enum => [qw(i6300esb ib700)],
 149         description => "Watchdog type to emulate.",
 150         default => 'i6300esb',
 151         optional => 1,
 152     },
 153     action => {
 154         type => 'string',
 155         enum => [qw(reset shutdown poweroff pause debug none)],
 156         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 157         optional => 1,
 158     },
 159 };
 160 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 161
 162 my $agent_fmt = {
 163     enabled => {
 164         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 165         type => 'boolean',
 166         default => 0,
 167         default_key => 1,
 168     },
 169     fstrim_cloned_disks => {
 170         description => "Run fstrim after moving a disk or migrating the VM.",
 171         type => 'boolean',
 172         optional => 1,
 173         default => 0
 174     },
 175     type => {
 176         description => "Select the agent type",
 177         type => 'string',
 178         default => 'virtio',
 179         optional => 1,
 180         enum => [qw(virtio isa)],
 181     },
 182 };
 183
 184 my $vga_fmt = {
 185     type => {
 186         description => "Select the VGA type.",
 187         type => 'string',
 188         default => 'std',
 189         optional => 1,
 190         default_key => 1,
 191         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 192     },
 193     memory => {
 194         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 195         type => 'integer',
 196         optional => 1,
 197         minimum => 4,
 198         maximum => 512,
 199     },
 200 };
 201
 202 my $ivshmem_fmt = {
 203     size => {
 204         type => 'integer',
 205         minimum => 1,
 206         description => "The size of the file in MB.",
 207     },
 208     name => {
 209         type => 'string',
 210         pattern => '[a-zA-Z0-9\-]+',
 211         optional => 1,
 212         format_description => 'string',
 213         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 214     },
 215 };
 216
 217 my $audio_fmt = {
 218     device => {
 219         type => 'string',
 220         enum => [qw(ich9-intel-hda intel-hda AC97)],
 221         description =>  "Configure an audio device."
 222     },
 223     driver =>  {
 224         type => 'string',
 225         enum => ['spice', 'none'],
 226         default => 'spice',
 227         optional => 1,
 228         description => "Driver backend for the audio device."
 229     },
 230 };
 231
 232 my $spice_enhancements_fmt = {
 233     foldersharing => {
 234         type => 'boolean',
 235         optional => 1,
 236         default => '0',
 237         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 238     },
 239     videostreaming =>  {
 240         type => 'string',
 241         enum => ['off', 'all', 'filter'],
 242         default => 'off',
 243         optional => 1,
 244         description => "Enable video streaming. Uses compression for detected video streams."
 245     },
 246 };
 247
 248 my $rng_fmt = {
 249     source => {
 250         type => 'string',
 251         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 252         default_key => 1,
 253         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 254             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 255             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 256             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 257             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 258             ." a hardware RNG from the host.",
 259     },
 260     max_bytes => {
 261         type => 'integer',
 262         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 263             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 264             ." `0` to disable limiting (potentially dangerous!).",
 265         optional => 1,
 266
 267         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 268         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 269         # reading from /dev/urandom
 270         default => 1024,
 271     },
 272     period => {
 273         type => 'integer',
 274         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 275             ." the guest to retrieve another 'max_bytes' of entropy.",
 276         optional => 1,
 277         default => 1000,
 278     },
 279 };
 280
 281 my $confdesc = {
 282     onboot => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Specifies whether a VM will be started during system bootup.",
 286         default => 0,
 287     },
 288     autostart => {
 289         optional => 1,
 290         type => 'boolean',
 291         description => "Automatic restart after crash (currently ignored).",
 292         default => 0,
 293     },
 294     hotplug => {
 295         optional => 1,
 296         type => 'string', format => 'pve-hotplug-features',
 297         description => "Selectively enable hotplug features. This is a comma separated list of"
 298             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 299             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 300         default => 'network,disk,usb',
 301     },
 302     reboot => {
 303         optional => 1,
 304         type => 'boolean',
 305         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 306         default => 1,
 307     },
 308     lock => {
 309         optional => 1,
 310         type => 'string',
 311         description => "Lock/unlock the VM.",
 312         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 313     },
 314     cpulimit => {
 315         optional => 1,
 316         type => 'number',
 317         description => "Limit of CPU usage.",
 318         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 319             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 320         minimum => 0,
 321         maximum => 128,
 322         default => 0,
 323     },
 324     cpuunits => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 328         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 329             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 330             ." weights of all the other running VMs.",
 331         minimum => 2,
 332         maximum => 262144,
 333         default => 'cgroup v1: 1024, cgroup v2: 100',
 334     },
 335     memory => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 339             ." you use the balloon device.",
 340         minimum => 16,
 341         default => 512,
 342     },
 343     balloon => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 347         minimum => 0,
 348     },
 349     shares => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 353             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 354             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 355         minimum => 0,
 356         maximum => 50000,
 357         default => 1000,
 358     },
 359     keyboard => {
 360         optional => 1,
 361         type => 'string',
 362         description => "Keyboard layout for VNC server. The default is read from the"
 363             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 364         enum => PVE::Tools::kvmkeymaplist(),
 365         default => undef,
 366     },
 367     name => {
 368         optional => 1,
 369         type => 'string', format => 'dns-name',
 370         description => "Set a name for the VM. Only used on the configuration web interface.",
 371     },
 372     scsihw => {
 373         optional => 1,
 374         type => 'string',
 375         description => "SCSI controller model",
 376         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 377         default => 'lsi',
 378     },
 379     description => {
 380         optional => 1,
 381         type => 'string',
 382         description => "Description for the VM. Shown in the web-interface VM's summary."
 383             ." This is saved as comment inside the configuration file.",
 384         maxLength => 1024 * 8,
 385     },
 386     ostype => {
 387         optional => 1,
 388         type => 'string',
 389         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 390         description => "Specify guest operating system.",
 391         verbose_description => <<EODESC,
 392 Specify guest operating system. This is used to enable special
 393 optimization/features for specific operating systems:
 394
 395 [horizontal]
 396 other;; unspecified OS
 397 wxp;; Microsoft Windows XP
 398 w2k;; Microsoft Windows 2000
 399 w2k3;; Microsoft Windows 2003
 400 w2k8;; Microsoft Windows 2008
 401 wvista;; Microsoft Windows Vista
 402 win7;; Microsoft Windows 7
 403 win8;; Microsoft Windows 8/2012/2012r2
 404 win10;; Microsoft Windows 10/2016/2019
 405 win11;; Microsoft Windows 11/2022
 406 l24;; Linux 2.4 Kernel
 407 l26;; Linux 2.6 - 5.X Kernel
 408 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 409 EODESC
 410     },
 411     boot => {
 412         optional => 1,
 413         type => 'string', format => 'pve-qm-boot',
 414         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 415             ." key or 'legacy=' is deprecated.",
 416     },
 417     bootdisk => {
 418         optional => 1,
 419         type => 'string', format => 'pve-qm-bootdisk',
 420         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 421         pattern => '(ide|sata|scsi|virtio)\d+',
 422     },
 423     smp => {
 424         optional => 1,
 425         type => 'integer',
 426         description => "The number of CPUs. Please use option -sockets instead.",
 427         minimum => 1,
 428         default => 1,
 429     },
 430     sockets => {
 431         optional => 1,
 432         type => 'integer',
 433         description => "The number of CPU sockets.",
 434         minimum => 1,
 435         default => 1,
 436     },
 437     cores => {
 438         optional => 1,
 439         type => 'integer',
 440         description => "The number of cores per socket.",
 441         minimum => 1,
 442         default => 1,
 443     },
 444     numa => {
 445         optional => 1,
 446         type => 'boolean',
 447         description => "Enable/disable NUMA.",
 448         default => 0,
 449     },
 450     hugepages => {
 451         optional => 1,
 452         type => 'string',
 453         description => "Enable/disable hugepages memory.",
 454         enum => [qw(any 2 1024)],
 455     },
 456     keephugepages => {
 457         optional => 1,
 458         type => 'boolean',
 459         default => 0,
 460         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 461             ." after VM shutdown and can be used for subsequent starts.",
 462     },
 463     vcpus => {
 464         optional => 1,
 465         type => 'integer',
 466         description => "Number of hotplugged vcpus.",
 467         minimum => 1,
 468         default => 0,
 469     },
 470     acpi => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable ACPI.",
 474         default => 1,
 475     },
 476     agent => {
 477         optional => 1,
 478         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 479         type => 'string',
 480         format => $agent_fmt,
 481     },
 482     kvm => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Enable/disable KVM hardware virtualization.",
 486         default => 1,
 487     },
 488     tdf => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable time drift fix.",
 492         default => 0,
 493     },
 494     localtime => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 498             ." the `ostype` indicates a Microsoft Windows OS.",
 499     },
 500     freeze => {
 501         optional => 1,
 502         type => 'boolean',
 503         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 504     },
 505     vga => {
 506         optional => 1,
 507         type => 'string', format => $vga_fmt,
 508         description => "Configure the VGA hardware.",
 509         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 510             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 511             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 512             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 513             ." display server. For win* OS you can select how many independent displays you want,"
 514             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 515             ." using a serial device as terminal.",
 516     },
 517     watchdog => {
 518         optional => 1,
 519         type => 'string', format => 'pve-qm-watchdog',
 520         description => "Create a virtual hardware watchdog device.",
 521         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 522             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 523             ." else the watchdog will reset the guest (or execute the respective action specified)",
 524     },
 525     startdate => {
 526         optional => 1,
 527         type => 'string',
 528         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 529         description => "Set the initial date of the real time clock. Valid format for date are:"
 530             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 531         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 532         default => 'now',
 533     },
 534     startup =>  get_standard_option('pve-startup-order'),
 535     template => {
 536         optional => 1,
 537         type => 'boolean',
 538         description => "Enable/disable Template.",
 539         default => 0,
 540     },
 541     args => {
 542         optional => 1,
 543         type => 'string',
 544         description => "Arbitrary arguments passed to kvm.",
 545         verbose_description => <<EODESCR,
 546 Arbitrary arguments passed to kvm, for example:
 547
 548 args: -no-reboot -no-hpet
 549
 550 NOTE: this option is for experts only.
 551 EODESCR
 552     },
 553     tablet => {
 554         optional => 1,
 555         type => 'boolean',
 556         default => 1,
 557         description => "Enable/disable the USB tablet device.",
 558         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 559             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 560             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 561             ." may consider disabling this to save some context switches. This is turned off by"
 562             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 563     },
 564     migrate_speed => {
 565         optional => 1,
 566         type => 'integer',
 567         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 568         minimum => 0,
 569         default => 0,
 570     },
 571     migrate_downtime => {
 572         optional => 1,
 573         type => 'number',
 574         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 575         minimum => 0,
 576         default => 0.1,
 577     },
 578     cdrom => {
 579         optional => 1,
 580         type => 'string', format => 'pve-qm-ide',
 581         typetext => '<volume>',
 582         description => "This is an alias for option -ide2",
 583     },
 584     cpu => {
 585         optional => 1,
 586         description => "Emulated CPU type.",
 587         type => 'string',
 588         format => 'pve-vm-cpu-conf',
 589     },
 590     parent => get_standard_option('pve-snapshot-name', {
 591         optional => 1,
 592         description => "Parent snapshot name. This is used internally, and should not be modified.",
 593     }),
 594     snaptime => {
 595         optional => 1,
 596         description => "Timestamp for snapshots.",
 597         type => 'integer',
 598         minimum => 0,
 599     },
 600     vmstate => {
 601         optional => 1,
 602         type => 'string', format => 'pve-volume-id',
 603         description => "Reference to a volume which stores the VM state. This is used internally"
 604             ." for snapshots.",
 605     },
 606     vmstatestorage => get_standard_option('pve-storage-id', {
 607         description => "Default storage for VM state volumes/files.",
 608         optional => 1,
 609     }),
 610     runningmachine => get_standard_option('pve-qemu-machine', {
 611         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 612             ." for snapshots.",
 613     }),
 614     runningcpu => {
 615         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 616             ." internally for snapshots.",
 617         optional => 1,
 618         type => 'string',
 619         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 620         format_description => 'QEMU -cpu parameter'
 621     },
 622     machine => get_standard_option('pve-qemu-machine'),
 623     arch => {
 624         description => "Virtual processor architecture. Defaults to the host.",
 625         optional => 1,
 626         type => 'string',
 627         enum => [qw(x86_64 aarch64)],
 628     },
 629     smbios1 => {
 630         description => "Specify SMBIOS type 1 fields.",
 631         type => 'string', format => 'pve-qm-smbios1',
 632         maxLength => 512,
 633         optional => 1,
 634     },
 635     protection => {
 636         optional => 1,
 637         type => 'boolean',
 638         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 639             ." remove disk operations.",
 640         default => 0,
 641     },
 642     bios => {
 643         optional => 1,
 644         type => 'string',
 645         enum => [ qw(seabios ovmf) ],
 646         description => "Select BIOS implementation.",
 647         default => 'seabios',
 648     },
 649     vmgenid => {
 650         type => 'string',
 651         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 652         format_description => 'UUID',
 653         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 654             ." to disable explicitly.",
 655         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 656             ." value identifier to the guest OS. This allows to notify the guest operating system"
 657             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 658             ." execution or creation from a template). The guest operating system notices the"
 659             ." change, and is then able to react as appropriate by marking its copies of"
 660             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 661             ."Note that auto-creation only works when done through API/CLI create or update methods"
 662             .", but not when manually editing the config file.",
 663         default => "1 (autogenerated)",
 664         optional => 1,
 665     },
 666     hookscript => {
 667         type => 'string',
 668         format => 'pve-volume-id',
 669         optional => 1,
 670         description => "Script that will be executed during various steps in the vms lifetime.",
 671     },
 672     ivshmem => {
 673         type => 'string',
 674         format => $ivshmem_fmt,
 675         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 676             ." the host.",
 677         optional => 1,
 678     },
 679     audio0 => {
 680         type => 'string',
 681         format => $audio_fmt,
 682         description => "Configure a audio device, useful in combination with QXL/Spice.",
 683         optional => 1
 684     },
 685     spice_enhancements => {
 686         type => 'string',
 687         format => $spice_enhancements_fmt,
 688         description => "Configure additional enhancements for SPICE.",
 689         optional => 1
 690     },
 691     tags => {
 692         type => 'string', format => 'pve-tag-list',
 693         description => 'Tags of the VM. This is only meta information.',
 694         optional => 1,
 695     },
 696     rng0 => {
 697         type => 'string',
 698         format => $rng_fmt,
 699         description => "Configure a VirtIO-based Random Number Generator.",
 700         optional => 1,
 701     },
 702 };
 703
 704 my $cicustom_fmt = {
 705     meta => {
 706         type => 'string',
 707         optional => 1,
 708         description => 'Specify a custom file containing all meta data passed to the VM via"
 709             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     network => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all network data passed to the VM via'
 717             .' cloud-init.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     user => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all user data passed to the VM via'
 725             .' cloud-init.',
 726         format => 'pve-volume-id',
 727         format_description => 'volume',
 728     },
 729 };
 730 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 731
 732 my $confdesc_cloudinit = {
 733     citype => {
 734         optional => 1,
 735         type => 'string',
 736         description => 'Specifies the cloud-init configuration format. The default depends on the'
 737             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 738             .' and `configdrive2` for windows.',
 739         enum => ['configdrive2', 'nocloud', 'opennebula'],
 740     },
 741     ciuser => {
 742         optional => 1,
 743         type => 'string',
 744         description => "cloud-init: User name to change ssh keys and password for instead of the"
 745             ." image's configured default user.",
 746     },
 747     cipassword => {
 748         optional => 1,
 749         type => 'string',
 750         description => 'cloud-init: Password to assign the user. Using this is generally not'
 751             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 752             .' support hashed passwords.',
 753     },
 754     cicustom => {
 755         optional => 1,
 756         type => 'string',
 757         description => 'cloud-init: Specify custom files to replace the automatically generated'
 758             .' ones at start.',
 759         format => 'pve-qm-cicustom',
 760     },
 761     searchdomain => {
 762         optional => 1,
 763         type => 'string',
 764         description => "cloud-init: Sets DNS search domains for a container. Create will'
 765             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 766             .' are set.",
 767     },
 768     nameserver => {
 769         optional => 1,
 770         type => 'string', format => 'address-list',
 771         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 772             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 773             .' are set.",
 774     },
 775     sshkeys => {
 776         optional => 1,
 777         type => 'string',
 778         format => 'urlencoded',
 779         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 780     },
 781 };
 782
 783 # what about other qemu settings ?
 784 #cpu => 'string',
 785 #machine => 'string',
 786 #fda => 'file',
 787 #fdb => 'file',
 788 #mtdblock => 'file',
 789 #sd => 'file',
 790 #pflash => 'file',
 791 #snapshot => 'bool',
 792 #bootp => 'file',
 793 ##tftp => 'dir',
 794 ##smb => 'dir',
 795 #kernel => 'file',
 796 #append => 'string',
 797 #initrd => 'file',
 798 ##soundhw => 'string',
 799
 800 while (my ($k, $v) = each %$confdesc) {
 801     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 802 }
 803
 804 my $MAX_USB_DEVICES = 5;
 805 my $MAX_NETS = 32;
 806 my $MAX_SERIAL_PORTS = 4;
 807 my $MAX_PARALLEL_PORTS = 3;
 808 my $MAX_NUMA = 8;
 809
 810 my $numa_fmt = {
 811     cpus => {
 812         type => "string",
 813         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 814         description => "CPUs accessing this NUMA node.",
 815         format_description => "id[-id];...",
 816     },
 817     memory => {
 818         type => "number",
 819         description => "Amount of memory this NUMA node provides.",
 820         optional => 1,
 821     },
 822     hostnodes => {
 823         type => "string",
 824         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 825         description => "Host NUMA nodes to use.",
 826         format_description => "id[-id];...",
 827         optional => 1,
 828     },
 829     policy => {
 830         type => 'string',
 831         enum => [qw(preferred bind interleave)],
 832         description => "NUMA allocation policy.",
 833         optional => 1,
 834     },
 835 };
 836 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 837 my $numadesc = {
 838     optional => 1,
 839     type => 'string', format => $numa_fmt,
 840     description => "NUMA topology.",
 841 };
 842 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 843
 844 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 845     $confdesc->{"numa$i"} = $numadesc;
 846 }
 847
 848 my $nic_model_list = [
 849     'e1000',
 850     'e1000-82540em',
 851     'e1000-82544gc',
 852     'e1000-82545em',
 853     'e1000e',
 854     'i82551',
 855     'i82557b',
 856     'i82559er',
 857     'ne2k_isa',
 858     'ne2k_pci',
 859     'pcnet',
 860     'rtl8139',
 861     'virtio',
 862     'vmxnet3',
 863 ];
 864 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 865
 866 my $net_fmt_bridge_descr = <<__EOD__;
 867 Bridge to attach the network device to. The Proxmox VE standard bridge
 868 is called 'vmbr0'.
 869
 870 If you do not specify a bridge, we create a kvm user (NATed) network
 871 device, which provides DHCP and DNS services. The following addresses
 872 are used:
 873
 874  10.0.2.2   Gateway
 875  10.0.2.3   DNS Server
 876  10.0.2.4   SMB Server
 877
 878 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 879 __EOD__
 880
 881 my $net_fmt = {
 882     macaddr  => get_standard_option('mac-addr', {
 883         description => "MAC address. That address must be unique withing your network. This is"
 884             ." automatically generated if not specified.",
 885     }),
 886     model => {
 887         type => 'string',
 888         description => "Network Card Model. The 'virtio' model provides the best performance with"
 889             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 890             ." best to use 'e1000'.",
 891         enum => $nic_model_list,
 892         default_key => 1,
 893     },
 894     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 895     bridge => {
 896         type => 'string',
 897         description => $net_fmt_bridge_descr,
 898         format_description => 'bridge',
 899         pattern => '[-_.\w\d]+',
 900         optional => 1,
 901     },
 902     queues => {
 903         type => 'integer',
 904         minimum => 0, maximum => 16,
 905         description => 'Number of packet queues to be used on the device.',
 906         optional => 1,
 907     },
 908     rate => {
 909         type => 'number',
 910         minimum => 0,
 911         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 912         optional => 1,
 913     },
 914     tag => {
 915         type => 'integer',
 916         minimum => 1, maximum => 4094,
 917         description => 'VLAN tag to apply to packets on this interface.',
 918         optional => 1,
 919     },
 920     trunks => {
 921         type => 'string',
 922         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 923         description => 'VLAN trunks to pass through this interface.',
 924         format_description => 'vlanid[;vlanid...]',
 925         optional => 1,
 926     },
 927     firewall => {
 928         type => 'boolean',
 929         description => 'Whether this interface should be protected by the firewall.',
 930         optional => 1,
 931     },
 932     link_down => {
 933         type => 'boolean',
 934         description => 'Whether this interface should be disconnected (like pulling the plug).',
 935         optional => 1,
 936     },
 937     mtu => {
 938         type => 'integer',
 939         minimum => 1, maximum => 65520,
 940         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 941         optional => 1,
 942     },
 943 };
 944
 945 my $netdesc = {
 946     optional => 1,
 947     type => 'string', format => $net_fmt,
 948     description => "Specify network devices.",
 949 };
 950
 951 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 952
 953 my $ipconfig_fmt = {
 954     ip => {
 955         type => 'string',
 956         format => 'pve-ipv4-config',
 957         format_description => 'IPv4Format/CIDR',
 958         description => 'IPv4 address in CIDR format.',
 959         optional => 1,
 960         default => 'dhcp',
 961     },
 962     gw => {
 963         type => 'string',
 964         format => 'ipv4',
 965         format_description => 'GatewayIPv4',
 966         description => 'Default gateway for IPv4 traffic.',
 967         optional => 1,
 968         requires => 'ip',
 969     },
 970     ip6 => {
 971         type => 'string',
 972         format => 'pve-ipv6-config',
 973         format_description => 'IPv6Format/CIDR',
 974         description => 'IPv6 address in CIDR format.',
 975         optional => 1,
 976         default => 'dhcp',
 977     },
 978     gw6 => {
 979         type => 'string',
 980         format => 'ipv6',
 981         format_description => 'GatewayIPv6',
 982         description => 'Default gateway for IPv6 traffic.',
 983         optional => 1,
 984         requires => 'ip6',
 985     },
 986 };
 987 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 988 my $ipconfigdesc = {
 989     optional => 1,
 990     type => 'string', format => 'pve-qm-ipconfig',
 991     description => <<'EODESCR',
 992 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 993
 994 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 995
 996 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 997 gateway should be provided.
 998 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 999 cloud-init 19.4 or newer.
1000
1001 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1002 dhcp on IPv4.
1003 EODESCR
1004 };
1005 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1006
1007 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1008     $confdesc->{"net$i"} = $netdesc;
1009     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1010 }
1011
1012 foreach my $key (keys %$confdesc_cloudinit) {
1013     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1014 }
1015
1016 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1017 sub verify_volume_id_or_qm_path {
1018     my ($volid, $noerr) = @_;
1019
1020     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1021         return $volid;
1022     }
1023
1024     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1025     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1026     if ($@) {
1027         return if $noerr;
1028         die $@;
1029     }
1030     return $volid;
1031 }
1032
1033 my $usb_fmt = {
1034     host => {
1035         default_key => 1,
1036         type => 'string', format => 'pve-qm-usb-device',
1037         format_description => 'HOSTUSBDEVICE|spice',
1038         description => <<EODESCR,
1039 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1040
1041  'bus-port(.port)*' (decimal numbers) or
1042  'vendor_id:product_id' (hexadeciaml numbers) or
1043  'spice'
1044
1045 You can use the 'lsusb -t' command to list existing usb devices.
1046
1047 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1048 machines - use with special care.
1049
1050 The value 'spice' can be used to add a usb redirection devices for spice.
1051 EODESCR
1052     },
1053     usb3 => {
1054         optional => 1,
1055         type => 'boolean',
1056         description => "Specifies whether if given host option is a USB3 device or port.",
1057         default => 0,
1058     },
1059 };
1060
1061 my $usbdesc = {
1062     optional => 1,
1063     type => 'string', format => $usb_fmt,
1064     description => "Configure an USB device (n is 0 to 4).",
1065 };
1066 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1067
1068 my $serialdesc = {
1069         optional => 1,
1070         type => 'string',
1071         pattern => '(/dev/.+|socket)',
1072         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1073         verbose_description =>  <<EODESCR,
1074 Create a serial device inside the VM (n is 0 to 3), and pass through a
1075 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1076 host side (use 'qm terminal' to open a terminal connection).
1077
1078 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1079 use with special care.
1080
1081 CAUTION: Experimental! User reported problems with this option.
1082 EODESCR
1083 };
1084
1085 my $paralleldesc= {
1086         optional => 1,
1087         type => 'string',
1088         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1089         description =>  "Map host parallel devices (n is 0 to 2).",
1090         verbose_description =>  <<EODESCR,
1091 Map host parallel devices (n is 0 to 2).
1092
1093 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1094 machines - use with special care.
1095
1096 CAUTION: Experimental! User reported problems with this option.
1097 EODESCR
1098 };
1099
1100 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1101     $confdesc->{"parallel$i"} = $paralleldesc;
1102 }
1103
1104 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1105     $confdesc->{"serial$i"} = $serialdesc;
1106 }
1107
1108 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1109     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1110 }
1111
1112 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1113     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1114 }
1115
1116 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1117     $confdesc->{"usb$i"} = $usbdesc;
1118 }
1119
1120 my $boot_fmt = {
1121     legacy => {
1122         optional => 1,
1123         default_key => 1,
1124         type => 'string',
1125         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1126                      . " Deprecated, use 'order=' instead.",
1127         pattern => '[acdn]{1,4}',
1128         format_description => "[acdn]{1,4}",
1129
1130         # note: this is also the fallback if boot: is not given at all
1131         default => 'cdn',
1132     },
1133     order => {
1134         optional => 1,
1135         type => 'string',
1136         format => 'pve-qm-bootdev-list',
1137         format_description => "device[;device...]",
1138         description => <<EODESC,
1139 The guest will attempt to boot from devices in the order they appear here.
1140
1141 Disks, optical drives and passed-through storage USB devices will be directly
1142 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1143 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1144
1145 Note that only devices in this list will be marked as bootable and thus loaded
1146 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1147 (e.g. software-raid), you need to specify all of them here.
1148
1149 Overrides the deprecated 'legacy=[acdn]*' value when given.
1150 EODESC
1151     },
1152 };
1153 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1154
1155 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1156 sub verify_bootdev {
1157     my ($dev, $noerr) = @_;
1158
1159     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1160     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1161
1162     my $check = sub {
1163         my ($base) = @_;
1164         return 0 if $dev !~ m/^$base\d+$/;
1165         return 0 if !$confdesc->{$dev};
1166         return 1;
1167     };
1168
1169     return $dev if $check->("net");
1170     return $dev if $check->("usb");
1171     return $dev if $check->("hostpci");
1172
1173     return if $noerr;
1174     die "invalid boot device '$dev'\n";
1175 }
1176
1177 sub print_bootorder {
1178     my ($devs) = @_;
1179     return "" if !@$devs;
1180     my $data = { order => join(';', @$devs) };
1181     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1182 }
1183
1184 my $kvm_api_version = 0;
1185
1186 sub kvm_version {
1187     return $kvm_api_version if $kvm_api_version;
1188
1189     open my $fh, '<', '/dev/kvm' or return;
1190
1191     # 0xae00 => KVM_GET_API_VERSION
1192     $kvm_api_version = ioctl($fh, 0xae00, 0);
1193     close($fh);
1194
1195     return $kvm_api_version;
1196 }
1197
1198 my $kvm_user_version = {};
1199 my $kvm_mtime = {};
1200
1201 sub kvm_user_version {
1202     my ($binary) = @_;
1203
1204     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1205     my $st = stat($binary);
1206
1207     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1208     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1209         $cachedmtime == $st->mtime;
1210
1211     $kvm_user_version->{$binary} = 'unknown';
1212     $kvm_mtime->{$binary} = $st->mtime;
1213
1214     my $code = sub {
1215         my $line = shift;
1216         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1217             $kvm_user_version->{$binary} = $2;
1218         }
1219     };
1220
1221     eval { run_command([$binary, '--version'], outfunc => $code); };
1222     warn $@ if $@;
1223
1224     return $kvm_user_version->{$binary};
1225
1226 }
1227 my sub extract_version {
1228     my ($machine_type, $version) = @_;
1229     $version = kvm_user_version() if !defined($version);
1230     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1231 }
1232
1233 sub kernel_has_vhost_net {
1234     return -c '/dev/vhost-net';
1235 }
1236
1237 sub option_exists {
1238     my $key = shift;
1239     return defined($confdesc->{$key});
1240 }
1241
1242 my $cdrom_path;
1243 sub get_cdrom_path {
1244
1245     return  $cdrom_path if $cdrom_path;
1246
1247     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1248     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1249     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1250 }
1251
1252 sub get_iso_path {
1253     my ($storecfg, $vmid, $cdrom) = @_;
1254
1255     if ($cdrom eq 'cdrom') {
1256         return get_cdrom_path();
1257     } elsif ($cdrom eq 'none') {
1258         return '';
1259     } elsif ($cdrom =~ m|^/|) {
1260         return $cdrom;
1261     } else {
1262         return PVE::Storage::path($storecfg, $cdrom);
1263     }
1264 }
1265
1266 # try to convert old style file names to volume IDs
1267 sub filename_to_volume_id {
1268     my ($vmid, $file, $media) = @_;
1269
1270      if (!($file eq 'none' || $file eq 'cdrom' ||
1271           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1272
1273         return if $file =~ m|/|;
1274
1275         if ($media && $media eq 'cdrom') {
1276             $file = "local:iso/$file";
1277         } else {
1278             $file = "local:$vmid/$file";
1279         }
1280     }
1281
1282     return $file;
1283 }
1284
1285 sub verify_media_type {
1286     my ($opt, $vtype, $media) = @_;
1287
1288     return if !$media;
1289
1290     my $etype;
1291     if ($media eq 'disk') {
1292         $etype = 'images';
1293     } elsif ($media eq 'cdrom') {
1294         $etype = 'iso';
1295     } else {
1296         die "internal error";
1297     }
1298
1299     return if ($vtype eq $etype);
1300
1301     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1302 }
1303
1304 sub cleanup_drive_path {
1305     my ($opt, $storecfg, $drive) = @_;
1306
1307     # try to convert filesystem paths to volume IDs
1308
1309     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1310         ($drive->{file} !~ m|^/dev/.+|) &&
1311         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1312         ($drive->{file} !~ m/^\d+$/)) {
1313         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1314         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1315             if !$vtype;
1316         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1317         verify_media_type($opt, $vtype, $drive->{media});
1318         $drive->{file} = $volid;
1319     }
1320
1321     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1322 }
1323
1324 sub parse_hotplug_features {
1325     my ($data) = @_;
1326
1327     my $res = {};
1328
1329     return $res if $data eq '0';
1330
1331     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1332
1333     foreach my $feature (PVE::Tools::split_list($data)) {
1334         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1335             $res->{$1} = 1;
1336         } else {
1337             die "invalid hotplug feature '$feature'\n";
1338         }
1339     }
1340     return $res;
1341 }
1342
1343 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1344 sub pve_verify_hotplug_features {
1345     my ($value, $noerr) = @_;
1346
1347     return $value if parse_hotplug_features($value);
1348
1349     return if $noerr;
1350
1351     die "unable to parse hotplug option\n";
1352 }
1353
1354 sub scsi_inquiry {
1355     my($fh, $noerr) = @_;
1356
1357     my $SG_IO = 0x2285;
1358     my $SG_GET_VERSION_NUM = 0x2282;
1359
1360     my $versionbuf = "\x00" x 8;
1361     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1362     if (!$ret) {
1363         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1364         return;
1365     }
1366     my $version = unpack("I", $versionbuf);
1367     if ($version < 30000) {
1368         die "scsi generic interface too old\n"  if !$noerr;
1369         return;
1370     }
1371
1372     my $buf = "\x00" x 36;
1373     my $sensebuf = "\x00" x 8;
1374     my $cmd = pack("C x3 C x1", 0x12, 36);
1375
1376     # see /usr/include/scsi/sg.h
1377     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1378
1379     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1380                       length($sensebuf), 0, length($buf), $buf,
1381                       $cmd, $sensebuf, 6000);
1382
1383     $ret = ioctl($fh, $SG_IO, $packet);
1384     if (!$ret) {
1385         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1386         return;
1387     }
1388
1389     my @res = unpack($sg_io_hdr_t, $packet);
1390     if ($res[17] || $res[18]) {
1391         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1392         return;
1393     }
1394
1395     my $res = {};
1396     (my $byte0, my $byte1, $res->{vendor},
1397      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1398
1399     $res->{removable} = $byte1 & 128 ? 1 : 0;
1400     $res->{type} = $byte0 & 31;
1401
1402     return $res;
1403 }
1404
1405 sub path_is_scsi {
1406     my ($path) = @_;
1407
1408     my $fh = IO::File->new("+<$path") || return;
1409     my $res = scsi_inquiry($fh, 1);
1410     close($fh);
1411
1412     return $res;
1413 }
1414
1415 sub print_tabletdevice_full {
1416     my ($conf, $arch) = @_;
1417
1418     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1419
1420     # we use uhci for old VMs because tablet driver was buggy in older qemu
1421     my $usbbus;
1422     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1423         $usbbus = 'ehci';
1424     } else {
1425         $usbbus = 'uhci';
1426     }
1427
1428     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1429 }
1430
1431 sub print_keyboarddevice_full {
1432     my ($conf, $arch, $machine) = @_;
1433
1434     return if $arch ne 'aarch64';
1435
1436     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1437 }
1438
1439 my sub get_drive_id {
1440     my ($drive) = @_;
1441     return "$drive->{interface}$drive->{index}";
1442 }
1443
1444 sub print_drivedevice_full {
1445     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1446
1447     my $device = '';
1448     my $maxdev = 0;
1449
1450     my $drive_id = get_drive_id($drive);
1451     if ($drive->{interface} eq 'virtio') {
1452         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1453         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1454         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1455     } elsif ($drive->{interface} eq 'scsi') {
1456
1457         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1458         my $unit = $drive->{index} % $maxdev;
1459         my $devicetype = 'hd';
1460         my $path = '';
1461         if (drive_is_cdrom($drive)) {
1462             $devicetype = 'cd';
1463         } else {
1464             if ($drive->{file} =~ m|^/|) {
1465                 $path = $drive->{file};
1466                 if (my $info = path_is_scsi($path)) {
1467                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1468                         $devicetype = 'block';
1469                     } elsif ($info->{type} == 1) { # tape
1470                         $devicetype = 'generic';
1471                     }
1472                 }
1473             } else {
1474                  $path = PVE::Storage::path($storecfg, $drive->{file});
1475             }
1476
1477             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1478             my $version = extract_version($machine_type, kvm_user_version());
1479             if ($path =~ m/^iscsi\:\/\// &&
1480                !min_version($version, 4, 1)) {
1481                 $devicetype = 'generic';
1482             }
1483         }
1484
1485         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1486             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1487         } else {
1488             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1489                 .",lun=$drive->{index}";
1490         }
1491         $device .= ",drive=drive-$drive_id,id=$drive_id";
1492
1493         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1494             $device .= ",rotation_rate=1";
1495         }
1496         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1497
1498     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1499         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1500         my $controller = int($drive->{index} / $maxdev);
1501         my $unit = $drive->{index} % $maxdev;
1502         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1503
1504         $device = "ide-$devicetype";
1505         if ($drive->{interface} eq 'ide') {
1506             $device .= ",bus=ide.$controller,unit=$unit";
1507         } else {
1508             $device .= ",bus=ahci$controller.$unit";
1509         }
1510         $device .= ",drive=drive-$drive_id,id=$drive_id";
1511
1512         if ($devicetype eq 'hd') {
1513             if (my $model = $drive->{model}) {
1514                 $model = URI::Escape::uri_unescape($model);
1515                 $device .= ",model=$model";
1516             }
1517             if ($drive->{ssd}) {
1518                 $device .= ",rotation_rate=1";
1519             }
1520         }
1521         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1522     } elsif ($drive->{interface} eq 'usb') {
1523         die "implement me";
1524         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1525     } else {
1526         die "unsupported interface type";
1527     }
1528
1529     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1530
1531     if (my $serial = $drive->{serial}) {
1532         $serial = URI::Escape::uri_unescape($serial);
1533         $device .= ",serial=$serial";
1534     }
1535
1536
1537     return $device;
1538 }
1539
1540 sub get_initiator_name {
1541     my $initiator;
1542
1543     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1544     while (defined(my $line = <$fh>)) {
1545         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1546         $initiator = $1;
1547         last;
1548     }
1549     $fh->close();
1550
1551     return $initiator;
1552 }
1553
1554 sub print_drive_commandline_full {
1555     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1556
1557     my $path;
1558     my $volid = $drive->{file};
1559     my $format = $drive->{format};
1560     my $drive_id = get_drive_id($drive);
1561
1562     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1563     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1564
1565     if (drive_is_cdrom($drive)) {
1566         $path = get_iso_path($storecfg, $vmid, $volid);
1567         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1568     } else {
1569         if ($storeid) {
1570             $path = PVE::Storage::path($storecfg, $volid);
1571             $format //= qemu_img_format($scfg, $volname);
1572         } else {
1573             $path = $volid;
1574             $format //= "raw";
1575         }
1576    }
1577
1578    my $is_rbd = $path =~ m/^rbd:/;
1579
1580     my $opts = '';
1581     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1582     foreach my $o (@qemu_drive_options) {
1583         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1584     }
1585
1586     # snapshot only accepts on|off
1587     if (defined($drive->{snapshot})) {
1588         my $v = $drive->{snapshot} ? 'on' : 'off';
1589         $opts .= ",snapshot=$v";
1590     }
1591
1592     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1593         my ($dir, $qmpname) = @$type;
1594         if (my $v = $drive->{"mbps$dir"}) {
1595             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1596         }
1597         if (my $v = $drive->{"mbps${dir}_max"}) {
1598             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1599         }
1600         if (my $v = $drive->{"bps${dir}_max_length"}) {
1601             $opts .= ",throttling.bps$qmpname-max-length=$v";
1602         }
1603         if (my $v = $drive->{"iops${dir}"}) {
1604             $opts .= ",throttling.iops$qmpname=$v";
1605         }
1606         if (my $v = $drive->{"iops${dir}_max"}) {
1607             $opts .= ",throttling.iops$qmpname-max=$v";
1608         }
1609         if (my $v = $drive->{"iops${dir}_max_length"}) {
1610             $opts .= ",throttling.iops$qmpname-max-length=$v";
1611         }
1612     }
1613
1614     if ($pbs_name) {
1615         $format = "rbd" if $is_rbd;
1616         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1617             if !$format;
1618         $opts .= ",format=alloc-track,file.driver=$format";
1619     } elsif ($format) {
1620         $opts .= ",format=$format";
1621     }
1622
1623     my $cache_direct = 0;
1624
1625     if (my $cache = $drive->{cache}) {
1626         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1627     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1628         $opts .= ",cache=none";
1629         $cache_direct = 1;
1630     }
1631
1632     # io_uring with cache mode writeback or writethrough on krbd will hang...
1633     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1634
1635     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1636     # sometimes, just plain disable...
1637     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1638
1639     if (!$drive->{aio}) {
1640         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1641             # io_uring supports all cache modes
1642             $opts .= ",aio=io_uring";
1643         } else {
1644             # aio native works only with O_DIRECT
1645             if($cache_direct) {
1646                 $opts .= ",aio=native";
1647             } else {
1648                 $opts .= ",aio=threads";
1649             }
1650         }
1651     }
1652
1653     if (!drive_is_cdrom($drive)) {
1654         my $detectzeroes;
1655         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1656             $detectzeroes = 'off';
1657         } elsif ($drive->{discard}) {
1658             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1659         } else {
1660             # This used to be our default with discard not being specified:
1661             $detectzeroes = 'on';
1662         }
1663
1664         # note: 'detect-zeroes' works per blockdev and we want it to persist
1665         # after the alloc-track is removed, so put it on 'file' directly
1666         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1667         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1668     }
1669
1670     if ($pbs_name) {
1671         $opts .= ",backing=$pbs_name";
1672         $opts .= ",auto-remove=on";
1673     }
1674
1675     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1676     my $file_param = "file";
1677     if ($pbs_name) {
1678         # non-rbd drivers require the underlying file to be a seperate block
1679         # node, so add a second .file indirection
1680         $file_param .= ".file" if !$is_rbd;
1681         $file_param .= ".filename";
1682     }
1683     my $pathinfo = $path ? "$file_param=$path," : '';
1684
1685     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1686 }
1687
1688 sub print_pbs_blockdev {
1689     my ($pbs_conf, $pbs_name) = @_;
1690     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1691     $blockdev .= ",repository=$pbs_conf->{repository}";
1692     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1693     $blockdev .= ",archive=$pbs_conf->{archive}";
1694     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1695     return $blockdev;
1696 }
1697
1698 sub print_netdevice_full {
1699     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1700
1701     my $device = $net->{model};
1702     if ($net->{model} eq 'virtio') {
1703          $device = 'virtio-net-pci';
1704      };
1705
1706     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1707     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1708     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1709         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1710         # and out of each queue plus one config interrupt and control vector queue
1711         my $vectors = $net->{queues} * 2 + 2;
1712         $tmpstr .= ",vectors=$vectors,mq=on";
1713     }
1714     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1715
1716     if (my $mtu = $net->{mtu}) {
1717         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1718             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1719             if ($mtu == 1) {
1720                  $mtu = $bridge_mtu;
1721             } elsif ($mtu < 576) {
1722                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1723             } elsif ($mtu > $bridge_mtu) {
1724                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1725             }
1726             $tmpstr .= ",host_mtu=$mtu";
1727         } else {
1728             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1729         }
1730     }
1731
1732     if ($use_old_bios_files) {
1733         my $romfile;
1734         if ($device eq 'virtio-net-pci') {
1735             $romfile = 'pxe-virtio.rom';
1736         } elsif ($device eq 'e1000') {
1737             $romfile = 'pxe-e1000.rom';
1738         } elsif ($device eq 'e1000e') {
1739             $romfile = 'pxe-e1000e.rom';
1740         } elsif ($device eq 'ne2k') {
1741             $romfile = 'pxe-ne2k_pci.rom';
1742         } elsif ($device eq 'pcnet') {
1743             $romfile = 'pxe-pcnet.rom';
1744         } elsif ($device eq 'rtl8139') {
1745             $romfile = 'pxe-rtl8139.rom';
1746         }
1747         $tmpstr .= ",romfile=$romfile" if $romfile;
1748     }
1749
1750     return $tmpstr;
1751 }
1752
1753 sub print_netdev_full {
1754     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1755
1756     my $i = '';
1757     if ($netid =~ m/^net(\d+)$/) {
1758         $i = int($1);
1759     }
1760
1761     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1762
1763     my $ifname = "tap${vmid}i$i";
1764
1765     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1766     die "interface name '$ifname' is too long (max 15 character)\n"
1767         if length($ifname) >= 16;
1768
1769     my $vhostparam = '';
1770     if (is_native($arch)) {
1771         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1772     }
1773
1774     my $vmname = $conf->{name} || "vm$vmid";
1775
1776     my $netdev = "";
1777     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1778
1779     if ($net->{bridge}) {
1780         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1781             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1782     } else {
1783         $netdev = "type=user,id=$netid,hostname=$vmname";
1784     }
1785
1786     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1787
1788     return $netdev;
1789 }
1790
1791 my $vga_map = {
1792     'cirrus' => 'cirrus-vga',
1793     'std' => 'VGA',
1794     'vmware' => 'vmware-svga',
1795     'virtio' => 'virtio-vga',
1796 };
1797
1798 sub print_vga_device {
1799     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1800
1801     my $type = $vga_map->{$vga->{type}};
1802     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1803         $type = 'virtio-gpu';
1804     }
1805     my $vgamem_mb = $vga->{memory};
1806
1807     my $max_outputs = '';
1808     if ($qxlnum) {
1809         $type = $id ? 'qxl' : 'qxl-vga';
1810
1811         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1812             # set max outputs so linux can have up to 4 qxl displays with one device
1813             if (min_version($machine_version, 4, 1)) {
1814                 $max_outputs = ",max_outputs=4";
1815             }
1816         }
1817     }
1818
1819     die "no devicetype for $vga->{type}\n" if !$type;
1820
1821     my $memory = "";
1822     if ($vgamem_mb) {
1823         if ($vga->{type} eq 'virtio') {
1824             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1825             $memory = ",max_hostmem=$bytes";
1826         } elsif ($qxlnum) {
1827             # from https://www.spice-space.org/multiple-monitors.html
1828             $memory = ",vgamem_mb=$vga->{memory}";
1829             my $ram = $vgamem_mb * 4;
1830             my $vram = $vgamem_mb * 2;
1831             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1832         } else {
1833             $memory = ",vgamem_mb=$vga->{memory}";
1834         }
1835     } elsif ($qxlnum && $id) {
1836         $memory = ",ram_size=67108864,vram_size=33554432";
1837     }
1838
1839     my $edidoff = "";
1840     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1841         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1842     }
1843
1844     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1845     my $vgaid = "vga" . ($id // '');
1846     my $pciaddr;
1847     if ($q35 && $vgaid eq 'vga') {
1848         # the first display uses pcie.0 bus on q35 machines
1849         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1850     } else {
1851         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1852     }
1853
1854     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1855 }
1856
1857 sub parse_number_sets {
1858     my ($set) = @_;
1859     my $res = [];
1860     foreach my $part (split(/;/, $set)) {
1861         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1862             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1863             push @$res, [ $1, $2 ];
1864         } else {
1865             die "invalid range: $part\n";
1866         }
1867     }
1868     return $res;
1869 }
1870
1871 sub parse_numa {
1872     my ($data) = @_;
1873
1874     my $res = parse_property_string($numa_fmt, $data);
1875     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1876     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1877     return $res;
1878 }
1879
1880 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1881 sub parse_net {
1882     my ($data) = @_;
1883
1884     my $res = eval { parse_property_string($net_fmt, $data) };
1885     if ($@) {
1886         warn $@;
1887         return;
1888     }
1889     if (!defined($res->{macaddr})) {
1890         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1891         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1892     }
1893     return $res;
1894 }
1895
1896 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1897 sub parse_ipconfig {
1898     my ($data) = @_;
1899
1900     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1901     if ($@) {
1902         warn $@;
1903         return;
1904     }
1905
1906     if ($res->{gw} && !$res->{ip}) {
1907         warn 'gateway specified without specifying an IP address';
1908         return;
1909     }
1910     if ($res->{gw6} && !$res->{ip6}) {
1911         warn 'IPv6 gateway specified without specifying an IPv6 address';
1912         return;
1913     }
1914     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1915         warn 'gateway specified together with DHCP';
1916         return;
1917     }
1918     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1919         # gw6 + auto/dhcp
1920         warn "IPv6 gateway specified together with $res->{ip6} address";
1921         return;
1922     }
1923
1924     if (!$res->{ip} && !$res->{ip6}) {
1925         return { ip => 'dhcp', ip6 => 'dhcp' };
1926     }
1927
1928     return $res;
1929 }
1930
1931 sub print_net {
1932     my $net = shift;
1933
1934     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1935 }
1936
1937 sub add_random_macs {
1938     my ($settings) = @_;
1939
1940     foreach my $opt (keys %$settings) {
1941         next if $opt !~ m/^net(\d+)$/;
1942         my $net = parse_net($settings->{$opt});
1943         next if !$net;
1944         $settings->{$opt} = print_net($net);
1945     }
1946 }
1947
1948 sub vm_is_volid_owner {
1949     my ($storecfg, $vmid, $volid) = @_;
1950
1951     if ($volid !~  m|^/|) {
1952         my ($path, $owner);
1953         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1954         if ($owner && ($owner == $vmid)) {
1955             return 1;
1956         }
1957     }
1958
1959     return;
1960 }
1961
1962 sub vmconfig_register_unused_drive {
1963     my ($storecfg, $vmid, $conf, $drive) = @_;
1964
1965     if (drive_is_cloudinit($drive)) {
1966         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1967         warn $@ if $@;
1968     } elsif (!drive_is_cdrom($drive)) {
1969         my $volid = $drive->{file};
1970         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1971             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1972         }
1973     }
1974 }
1975
1976 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1977 my $smbios1_fmt = {
1978     uuid => {
1979         type => 'string',
1980         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1981         format_description => 'UUID',
1982         description => "Set SMBIOS1 UUID.",
1983         optional => 1,
1984     },
1985     version => {
1986         type => 'string',
1987         pattern => '[A-Za-z0-9+\/]+={0,2}',
1988         format_description => 'Base64 encoded string',
1989         description => "Set SMBIOS1 version.",
1990         optional => 1,
1991     },
1992     serial => {
1993         type => 'string',
1994         pattern => '[A-Za-z0-9+\/]+={0,2}',
1995         format_description => 'Base64 encoded string',
1996         description => "Set SMBIOS1 serial number.",
1997         optional => 1,
1998     },
1999     manufacturer => {
2000         type => 'string',
2001         pattern => '[A-Za-z0-9+\/]+={0,2}',
2002         format_description => 'Base64 encoded string',
2003         description => "Set SMBIOS1 manufacturer.",
2004         optional => 1,
2005     },
2006     product => {
2007         type => 'string',
2008         pattern => '[A-Za-z0-9+\/]+={0,2}',
2009         format_description => 'Base64 encoded string',
2010         description => "Set SMBIOS1 product ID.",
2011         optional => 1,
2012     },
2013     sku => {
2014         type => 'string',
2015         pattern => '[A-Za-z0-9+\/]+={0,2}',
2016         format_description => 'Base64 encoded string',
2017         description => "Set SMBIOS1 SKU string.",
2018         optional => 1,
2019     },
2020     family => {
2021         type => 'string',
2022         pattern => '[A-Za-z0-9+\/]+={0,2}',
2023         format_description => 'Base64 encoded string',
2024         description => "Set SMBIOS1 family string.",
2025         optional => 1,
2026     },
2027     base64 => {
2028         type => 'boolean',
2029         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2030         optional => 1,
2031     },
2032 };
2033
2034 sub parse_smbios1 {
2035     my ($data) = @_;
2036
2037     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2038     warn $@ if $@;
2039     return $res;
2040 }
2041
2042 sub print_smbios1 {
2043     my ($smbios1) = @_;
2044     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2045 }
2046
2047 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2048
2049 sub parse_watchdog {
2050     my ($value) = @_;
2051
2052     return if !$value;
2053
2054     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2055     warn $@ if $@;
2056     return $res;
2057 }
2058
2059 sub parse_guest_agent {
2060     my ($conf) = @_;
2061
2062     return {} if !defined($conf->{agent});
2063
2064     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2065     warn $@ if $@;
2066
2067     # if the agent is disabled ignore the other potentially set properties
2068     return {} if !$res->{enabled};
2069     return $res;
2070 }
2071
2072 sub get_qga_key {
2073     my ($conf, $key) = @_;
2074     return undef if !defined($conf->{agent});
2075
2076     my $agent = parse_guest_agent($conf);
2077     return $agent->{$key};
2078 }
2079
2080 sub parse_vga {
2081     my ($value) = @_;
2082
2083     return {} if !$value;
2084     my $res = eval { parse_property_string($vga_fmt, $value) };
2085     warn $@ if $@;
2086     return $res;
2087 }
2088
2089 sub parse_rng {
2090     my ($value) = @_;
2091
2092     return if !$value;
2093
2094     my $res = eval { parse_property_string($rng_fmt, $value) };
2095     warn $@ if $@;
2096     return $res;
2097 }
2098
2099 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2100 sub verify_usb_device {
2101     my ($value, $noerr) = @_;
2102
2103     return $value if parse_usb_device($value);
2104
2105     return if $noerr;
2106
2107     die "unable to parse usb device\n";
2108 }
2109
2110 # add JSON properties for create and set function
2111 sub json_config_properties {
2112     my $prop = shift;
2113
2114     my $skip_json_config_opts = {
2115         parent => 1,
2116         snaptime => 1,
2117         vmstate => 1,
2118         runningmachine => 1,
2119         runningcpu => 1,
2120     };
2121
2122     foreach my $opt (keys %$confdesc) {
2123         next if $skip_json_config_opts->{$opt};
2124         $prop->{$opt} = $confdesc->{$opt};
2125     }
2126
2127     return $prop;
2128 }
2129
2130 # return copy of $confdesc_cloudinit to generate documentation
2131 sub cloudinit_config_properties {
2132
2133     return dclone($confdesc_cloudinit);
2134 }
2135
2136 sub check_type {
2137     my ($key, $value) = @_;
2138
2139     die "unknown setting '$key'\n" if !$confdesc->{$key};
2140
2141     my $type = $confdesc->{$key}->{type};
2142
2143     if (!defined($value)) {
2144         die "got undefined value\n";
2145     }
2146
2147     if ($value =~ m/[\n\r]/) {
2148         die "property contains a line feed\n";
2149     }
2150
2151     if ($type eq 'boolean') {
2152         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2153         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2154         die "type check ('boolean') failed - got '$value'\n";
2155     } elsif ($type eq 'integer') {
2156         return int($1) if $value =~ m/^(\d+)$/;
2157         die "type check ('integer') failed - got '$value'\n";
2158     } elsif ($type eq 'number') {
2159         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2160         die "type check ('number') failed - got '$value'\n";
2161     } elsif ($type eq 'string') {
2162         if (my $fmt = $confdesc->{$key}->{format}) {
2163             PVE::JSONSchema::check_format($fmt, $value);
2164             return $value;
2165         }
2166         $value =~ s/^\"(.*)\"$/$1/;
2167         return $value;
2168     } else {
2169         die "internal error"
2170     }
2171 }
2172
2173 sub destroy_vm {
2174     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2175
2176     my $conf = PVE::QemuConfig->load_config($vmid);
2177
2178     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2179
2180     if ($conf->{template}) {
2181         # check if any base image is still used by a linked clone
2182         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2183                 my ($ds, $drive) = @_;
2184                 return if drive_is_cdrom($drive);
2185
2186                 my $volid = $drive->{file};
2187                 return if !$volid || $volid =~ m|^/|;
2188
2189                 die "base volume '$volid' is still in use by linked cloned\n"
2190                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2191
2192         });
2193     }
2194
2195     my $volids = {};
2196     my $remove_owned_drive = sub {
2197         my ($ds, $drive) = @_;
2198         return if drive_is_cdrom($drive, 1);
2199
2200         my $volid = $drive->{file};
2201         return if !$volid || $volid =~ m|^/|;
2202         return if $volids->{$volid};
2203
2204         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2205         return if !$path || !$owner || ($owner != $vmid);
2206
2207         $volids->{$volid} = 1;
2208         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2209         warn "Could not remove disk '$volid', check manually: $@" if $@;
2210     };
2211
2212     # only remove disks owned by this VM (referenced in the config)
2213     my $include_opts = {
2214         include_unused => 1,
2215         extra_keys => ['vmstate'],
2216     };
2217     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2218
2219     for my $snap (values %{$conf->{snapshots}}) {
2220         next if !defined($snap->{vmstate});
2221         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2222         next if !defined($drive);
2223         $remove_owned_drive->('vmstate', $drive);
2224     }
2225
2226     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2227
2228     if ($purge_unreferenced) { # also remove unreferenced disk
2229         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2230         PVE::Storage::foreach_volid($vmdisks, sub {
2231             my ($volid, $sid, $volname, $d) = @_;
2232             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2233             warn $@ if $@;
2234         });
2235     }
2236
2237     if (defined $replacement_conf) {
2238         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2239     } else {
2240         PVE::QemuConfig->destroy_config($vmid);
2241     }
2242 }
2243
2244 sub parse_vm_config {
2245     my ($filename, $raw) = @_;
2246
2247     return if !defined($raw);
2248
2249     my $res = {
2250         digest => Digest::SHA::sha1_hex($raw),
2251         snapshots => {},
2252         pending => {},
2253     };
2254
2255     $filename =~ m|/qemu-server/(\d+)\.conf$|
2256         || die "got strange filename '$filename'";
2257
2258     my $vmid = $1;
2259
2260     my $conf = $res;
2261     my $descr;
2262     my $section = '';
2263
2264     my @lines = split(/\n/, $raw);
2265     foreach my $line (@lines) {
2266         next if $line =~ m/^\s*$/;
2267
2268         if ($line =~ m/^\[PENDING\]\s*$/i) {
2269             $section = 'pending';
2270             if (defined($descr)) {
2271                 $descr =~ s/\s+$//;
2272                 $conf->{description} = $descr;
2273             }
2274             $descr = undef;
2275             $conf = $res->{$section} = {};
2276             next;
2277
2278         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2279             $section = $1;
2280             if (defined($descr)) {
2281                 $descr =~ s/\s+$//;
2282                 $conf->{description} = $descr;
2283             }
2284             $descr = undef;
2285             $conf = $res->{snapshots}->{$section} = {};
2286             next;
2287         }
2288
2289         if ($line =~ m/^\#(.*)\s*$/) {
2290             $descr = '' if !defined($descr);
2291             $descr .= PVE::Tools::decode_text($1) . "\n";
2292             next;
2293         }
2294
2295         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2296             $descr = '' if !defined($descr);
2297             $descr .= PVE::Tools::decode_text($2);
2298         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2299             $conf->{snapstate} = $1;
2300         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2301             my $key = $1;
2302             my $value = $2;
2303             $conf->{$key} = $value;
2304         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2305             my $value = $1;
2306             if ($section eq 'pending') {
2307                 $conf->{delete} = $value; # we parse this later
2308             } else {
2309                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2310             }
2311         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2312             my $key = $1;
2313             my $value = $2;
2314             eval { $value = check_type($key, $value); };
2315             if ($@) {
2316                 warn "vm $vmid - unable to parse value of '$key' - $@";
2317             } else {
2318                 $key = 'ide2' if $key eq 'cdrom';
2319                 my $fmt = $confdesc->{$key}->{format};
2320                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2321                     my $v = parse_drive($key, $value);
2322                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2323                         $v->{file} = $volid;
2324                         $value = print_drive($v);
2325                     } else {
2326                         warn "vm $vmid - unable to parse value of '$key'\n";
2327                         next;
2328                     }
2329                 }
2330
2331                 $conf->{$key} = $value;
2332             }
2333         } else {
2334             warn "vm $vmid - unable to parse config: $line\n";
2335         }
2336     }
2337
2338     if (defined($descr)) {
2339         $descr =~ s/\s+$//;
2340         $conf->{description} = $descr;
2341     }
2342     delete $res->{snapstate}; # just to be sure
2343
2344     return $res;
2345 }
2346
2347 sub write_vm_config {
2348     my ($filename, $conf) = @_;
2349
2350     delete $conf->{snapstate}; # just to be sure
2351
2352     if ($conf->{cdrom}) {
2353         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2354         $conf->{ide2} = $conf->{cdrom};
2355         delete $conf->{cdrom};
2356     }
2357
2358     # we do not use 'smp' any longer
2359     if ($conf->{sockets}) {
2360         delete $conf->{smp};
2361     } elsif ($conf->{smp}) {
2362         $conf->{sockets} = $conf->{smp};
2363         delete $conf->{cores};
2364         delete $conf->{smp};
2365     }
2366
2367     my $used_volids = {};
2368
2369     my $cleanup_config = sub {
2370         my ($cref, $pending, $snapname) = @_;
2371
2372         foreach my $key (keys %$cref) {
2373             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2374                 $key eq 'snapstate' || $key eq 'pending';
2375             my $value = $cref->{$key};
2376             if ($key eq 'delete') {
2377                 die "propertry 'delete' is only allowed in [PENDING]\n"
2378                     if !$pending;
2379                 # fixme: check syntax?
2380                 next;
2381             }
2382             eval { $value = check_type($key, $value); };
2383             die "unable to parse value of '$key' - $@" if $@;
2384
2385             $cref->{$key} = $value;
2386
2387             if (!$snapname && is_valid_drivename($key)) {
2388                 my $drive = parse_drive($key, $value);
2389                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2390             }
2391         }
2392     };
2393
2394     &$cleanup_config($conf);
2395
2396     &$cleanup_config($conf->{pending}, 1);
2397
2398     foreach my $snapname (keys %{$conf->{snapshots}}) {
2399         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2400         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2401     }
2402
2403     # remove 'unusedX' settings if we re-add a volume
2404     foreach my $key (keys %$conf) {
2405         my $value = $conf->{$key};
2406         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2407             delete $conf->{$key};
2408         }
2409     }
2410
2411     my $generate_raw_config = sub {
2412         my ($conf, $pending) = @_;
2413
2414         my $raw = '';
2415
2416         # add description as comment to top of file
2417         if (defined(my $descr = $conf->{description})) {
2418             if ($descr) {
2419                 foreach my $cl (split(/\n/, $descr)) {
2420                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2421                 }
2422             } else {
2423                 $raw .= "#\n" if $pending;
2424             }
2425         }
2426
2427         foreach my $key (sort keys %$conf) {
2428             next if $key =~ /^(digest|description|pending|snapshots)$/;
2429             $raw .= "$key: $conf->{$key}\n";
2430         }
2431         return $raw;
2432     };
2433
2434     my $raw = &$generate_raw_config($conf);
2435
2436     if (scalar(keys %{$conf->{pending}})){
2437         $raw .= "\n[PENDING]\n";
2438         $raw .= &$generate_raw_config($conf->{pending}, 1);
2439     }
2440
2441     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2442         $raw .= "\n[$snapname]\n";
2443         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2444     }
2445
2446     return $raw;
2447 }
2448
2449 sub load_defaults {
2450
2451     my $res = {};
2452
2453     # we use static defaults from our JSON schema configuration
2454     foreach my $key (keys %$confdesc) {
2455         if (defined(my $default = $confdesc->{$key}->{default})) {
2456             $res->{$key} = $default;
2457         }
2458     }
2459
2460     return $res;
2461 }
2462
2463 sub config_list {
2464     my $vmlist = PVE::Cluster::get_vmlist();
2465     my $res = {};
2466     return $res if !$vmlist || !$vmlist->{ids};
2467     my $ids = $vmlist->{ids};
2468     my $nodename = nodename();
2469
2470     foreach my $vmid (keys %$ids) {
2471         my $d = $ids->{$vmid};
2472         next if !$d->{node} || $d->{node} ne $nodename;
2473         next if !$d->{type} || $d->{type} ne 'qemu';
2474         $res->{$vmid}->{exists} = 1;
2475     }
2476     return $res;
2477 }
2478
2479 # test if VM uses local resources (to prevent migration)
2480 sub check_local_resources {
2481     my ($conf, $noerr) = @_;
2482
2483     my @loc_res = ();
2484
2485     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2486     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2487
2488     push @loc_res, "ivshmem" if $conf->{ivshmem};
2489
2490     foreach my $k (keys %$conf) {
2491         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2492         # sockets are safe: they will recreated be on the target side post-migrate
2493         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2494         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2495     }
2496
2497     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2498
2499     return \@loc_res;
2500 }
2501
2502 # check if used storages are available on all nodes (use by migrate)
2503 sub check_storage_availability {
2504     my ($storecfg, $conf, $node) = @_;
2505
2506     PVE::QemuConfig->foreach_volume($conf, sub {
2507         my ($ds, $drive) = @_;
2508
2509         my $volid = $drive->{file};
2510         return if !$volid;
2511
2512         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2513         return if !$sid;
2514
2515         # check if storage is available on both nodes
2516         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2517         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2518
2519         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2520
2521         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2522             if !$scfg->{content}->{$vtype};
2523    });
2524 }
2525
2526 # list nodes where all VM images are available (used by has_feature API)
2527 sub shared_nodes {
2528     my ($conf, $storecfg) = @_;
2529
2530     my $nodelist = PVE::Cluster::get_nodelist();
2531     my $nodehash = { map { $_ => 1 } @$nodelist };
2532     my $nodename = nodename();
2533
2534     PVE::QemuConfig->foreach_volume($conf, sub {
2535         my ($ds, $drive) = @_;
2536
2537         my $volid = $drive->{file};
2538         return if !$volid;
2539
2540         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2541         if ($storeid) {
2542             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2543             if ($scfg->{disable}) {
2544                 $nodehash = {};
2545             } elsif (my $avail = $scfg->{nodes}) {
2546                 foreach my $node (keys %$nodehash) {
2547                     delete $nodehash->{$node} if !$avail->{$node};
2548                 }
2549             } elsif (!$scfg->{shared}) {
2550                 foreach my $node (keys %$nodehash) {
2551                     delete $nodehash->{$node} if $node ne $nodename
2552                 }
2553             }
2554         }
2555     });
2556
2557     return $nodehash
2558 }
2559
2560 sub check_local_storage_availability {
2561     my ($conf, $storecfg) = @_;
2562
2563     my $nodelist = PVE::Cluster::get_nodelist();
2564     my $nodehash = { map { $_ => {} } @$nodelist };
2565
2566     PVE::QemuConfig->foreach_volume($conf, sub {
2567         my ($ds, $drive) = @_;
2568
2569         my $volid = $drive->{file};
2570         return if !$volid;
2571
2572         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2573         if ($storeid) {
2574             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2575
2576             if ($scfg->{disable}) {
2577                 foreach my $node (keys %$nodehash) {
2578                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2579                 }
2580             } elsif (my $avail = $scfg->{nodes}) {
2581                 foreach my $node (keys %$nodehash) {
2582                     if (!$avail->{$node}) {
2583                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2584                     }
2585                 }
2586             }
2587         }
2588     });
2589
2590     foreach my $node (values %$nodehash) {
2591         if (my $unavail = $node->{unavailable_storages}) {
2592             $node->{unavailable_storages} = [ sort keys %$unavail ];
2593         }
2594     }
2595
2596     return $nodehash
2597 }
2598
2599 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2600 sub check_running {
2601     my ($vmid, $nocheck, $node) = @_;
2602
2603     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2604     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2605 }
2606
2607 sub vzlist {
2608
2609     my $vzlist = config_list();
2610
2611     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2612
2613     while (defined(my $de = $fd->read)) {
2614         next if $de !~ m/^(\d+)\.pid$/;
2615         my $vmid = $1;
2616         next if !defined($vzlist->{$vmid});
2617         if (my $pid = check_running($vmid)) {
2618             $vzlist->{$vmid}->{pid} = $pid;
2619         }
2620     }
2621
2622     return $vzlist;
2623 }
2624
2625 our $vmstatus_return_properties = {
2626     vmid => get_standard_option('pve-vmid'),
2627     status => {
2628         description => "Qemu process status.",
2629         type => 'string',
2630         enum => ['stopped', 'running'],
2631     },
2632     maxmem => {
2633         description => "Maximum memory in bytes.",
2634         type => 'integer',
2635         optional => 1,
2636         renderer => 'bytes',
2637     },
2638     maxdisk => {
2639         description => "Root disk size in bytes.",
2640         type => 'integer',
2641         optional => 1,
2642         renderer => 'bytes',
2643     },
2644     name => {
2645         description => "VM name.",
2646         type => 'string',
2647         optional => 1,
2648     },
2649     qmpstatus => {
2650         description => "Qemu QMP agent status.",
2651         type => 'string',
2652         optional => 1,
2653     },
2654     pid => {
2655         description => "PID of running qemu process.",
2656         type => 'integer',
2657         optional => 1,
2658     },
2659     uptime => {
2660         description => "Uptime.",
2661         type => 'integer',
2662         optional => 1,
2663         renderer => 'duration',
2664     },
2665     cpus => {
2666         description => "Maximum usable CPUs.",
2667         type => 'number',
2668         optional => 1,
2669     },
2670     lock => {
2671         description => "The current config lock, if any.",
2672         type => 'string',
2673         optional => 1,
2674     },
2675     tags => {
2676         description  => "The current configured tags, if any",
2677         type => 'string',
2678         optional => 1,
2679     },
2680     'running-machine' => {
2681         description => "The currently running machine type (if running).",
2682         type => 'string',
2683         optional => 1,
2684     },
2685     'running-qemu' => {
2686         description => "The currently running QEMU version (if running).",
2687         type => 'string',
2688         optional => 1,
2689     },
2690 };
2691
2692 my $last_proc_pid_stat;
2693
2694 # get VM status information
2695 # This must be fast and should not block ($full == false)
2696 # We only query KVM using QMP if $full == true (this can be slow)
2697 sub vmstatus {
2698     my ($opt_vmid, $full) = @_;
2699
2700     my $res = {};
2701
2702     my $storecfg = PVE::Storage::config();
2703
2704     my $list = vzlist();
2705     my $defaults = load_defaults();
2706
2707     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2708
2709     my $cpucount = $cpuinfo->{cpus} || 1;
2710
2711     foreach my $vmid (keys %$list) {
2712         next if $opt_vmid && ($vmid ne $opt_vmid);
2713
2714         my $conf = PVE::QemuConfig->load_config($vmid);
2715
2716         my $d = { vmid => int($vmid) };
2717         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2718
2719         # fixme: better status?
2720         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2721
2722         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2723         if (defined($size)) {
2724             $d->{disk} = 0; # no info available
2725             $d->{maxdisk} = $size;
2726         } else {
2727             $d->{disk} = 0;
2728             $d->{maxdisk} = 0;
2729         }
2730
2731         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2732             * ($conf->{cores} || $defaults->{cores});
2733         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2734         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2735
2736         $d->{name} = $conf->{name} || "VM $vmid";
2737         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2738             : $defaults->{memory}*(1024*1024);
2739
2740         if ($conf->{balloon}) {
2741             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2742             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2743                 : $defaults->{shares};
2744         }
2745
2746         $d->{uptime} = 0;
2747         $d->{cpu} = 0;
2748         $d->{mem} = 0;
2749
2750         $d->{netout} = 0;
2751         $d->{netin} = 0;
2752
2753         $d->{diskread} = 0;
2754         $d->{diskwrite} = 0;
2755
2756         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2757
2758         $d->{serial} = 1 if conf_has_serial($conf);
2759         $d->{lock} = $conf->{lock} if $conf->{lock};
2760         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2761
2762         $res->{$vmid} = $d;
2763     }
2764
2765     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2766     foreach my $dev (keys %$netdev) {
2767         next if $dev !~ m/^tap([1-9]\d*)i/;
2768         my $vmid = $1;
2769         my $d = $res->{$vmid};
2770         next if !$d;
2771
2772         $d->{netout} += $netdev->{$dev}->{receive};
2773         $d->{netin} += $netdev->{$dev}->{transmit};
2774
2775         if ($full) {
2776             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2777             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2778         }
2779
2780     }
2781
2782     my $ctime = gettimeofday;
2783
2784     foreach my $vmid (keys %$list) {
2785
2786         my $d = $res->{$vmid};
2787         my $pid = $d->{pid};
2788         next if !$pid;
2789
2790         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2791         next if !$pstat; # not running
2792
2793         my $used = $pstat->{utime} + $pstat->{stime};
2794
2795         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2796
2797         if ($pstat->{vsize}) {
2798             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2799         }
2800
2801         my $old = $last_proc_pid_stat->{$pid};
2802         if (!$old) {
2803             $last_proc_pid_stat->{$pid} = {
2804                 time => $ctime,
2805                 used => $used,
2806                 cpu => 0,
2807             };
2808             next;
2809         }
2810
2811         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2812
2813         if ($dtime > 1000) {
2814             my $dutime = $used -  $old->{used};
2815
2816             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2817             $last_proc_pid_stat->{$pid} = {
2818                 time => $ctime,
2819                 used => $used,
2820                 cpu => $d->{cpu},
2821             };
2822         } else {
2823             $d->{cpu} = $old->{cpu};
2824         }
2825     }
2826
2827     return $res if !$full;
2828
2829     my $qmpclient = PVE::QMPClient->new();
2830
2831     my $ballooncb = sub {
2832         my ($vmid, $resp) = @_;
2833
2834         my $info = $resp->{'return'};
2835         return if !$info->{max_mem};
2836
2837         my $d = $res->{$vmid};
2838
2839         # use memory assigned to VM
2840         $d->{maxmem} = $info->{max_mem};
2841         $d->{balloon} = $info->{actual};
2842
2843         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2844             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2845             $d->{freemem} = $info->{free_mem};
2846         }
2847
2848         $d->{ballooninfo} = $info;
2849     };
2850
2851     my $blockstatscb = sub {
2852         my ($vmid, $resp) = @_;
2853         my $data = $resp->{'return'} || [];
2854         my $totalrdbytes = 0;
2855         my $totalwrbytes = 0;
2856
2857         for my $blockstat (@$data) {
2858             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2859             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2860
2861             $blockstat->{device} =~ s/drive-//;
2862             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2863         }
2864         $res->{$vmid}->{diskread} = $totalrdbytes;
2865         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2866     };
2867
2868     my $machinecb = sub {
2869         my ($vmid, $resp) = @_;
2870         my $data = $resp->{'return'} || [];
2871
2872         $res->{$vmid}->{'running-machine'} =
2873             PVE::QemuServer::Machine::current_from_query_machines($data);
2874     };
2875
2876     my $versioncb = sub {
2877         my ($vmid, $resp) = @_;
2878         my $data = $resp->{'return'} // {};
2879         my $version = 'unknown';
2880
2881         if (my $v = $data->{qemu}) {
2882             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2883         }
2884
2885         $res->{$vmid}->{'running-qemu'} = $version;
2886     };
2887
2888     my $statuscb = sub {
2889         my ($vmid, $resp) = @_;
2890
2891         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2892         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2893         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2894         # this fails if ballon driver is not loaded, so this must be
2895         # the last commnand (following command are aborted if this fails).
2896         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2897
2898         my $status = 'unknown';
2899         if (!defined($status = $resp->{'return'}->{status})) {
2900             warn "unable to get VM status\n";
2901             return;
2902         }
2903
2904         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2905     };
2906
2907     foreach my $vmid (keys %$list) {
2908         next if $opt_vmid && ($vmid ne $opt_vmid);
2909         next if !$res->{$vmid}->{pid}; # not running
2910         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2911     }
2912
2913     $qmpclient->queue_execute(undef, 2);
2914
2915     foreach my $vmid (keys %$list) {
2916         next if $opt_vmid && ($vmid ne $opt_vmid);
2917         next if !$res->{$vmid}->{pid}; #not running
2918
2919         # we can't use the $qmpclient since it might have already aborted on
2920         # 'query-balloon', but this might also fail for older versions...
2921         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2922         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2923     }
2924
2925     foreach my $vmid (keys %$list) {
2926         next if $opt_vmid && ($vmid ne $opt_vmid);
2927         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2928     }
2929
2930     return $res;
2931 }
2932
2933 sub conf_has_serial {
2934     my ($conf) = @_;
2935
2936     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2937         if ($conf->{"serial$i"}) {
2938             return 1;
2939         }
2940     }
2941
2942     return 0;
2943 }
2944
2945 sub conf_has_audio {
2946     my ($conf, $id) = @_;
2947
2948     $id //= 0;
2949     my $audio = $conf->{"audio$id"};
2950     return if !defined($audio);
2951
2952     my $audioproperties = parse_property_string($audio_fmt, $audio);
2953     my $audiodriver = $audioproperties->{driver} // 'spice';
2954
2955     return {
2956         dev => $audioproperties->{device},
2957         dev_id => "audiodev$id",
2958         backend => $audiodriver,
2959         backend_id => "$audiodriver-backend${id}",
2960     };
2961 }
2962
2963 sub audio_devs {
2964     my ($audio, $audiopciaddr, $machine_version) = @_;
2965
2966     my $devs = [];
2967
2968     my $id = $audio->{dev_id};
2969     my $audiodev = "";
2970     if (min_version($machine_version, 4, 2)) {
2971         $audiodev = ",audiodev=$audio->{backend_id}";
2972     }
2973
2974     if ($audio->{dev} eq 'AC97') {
2975         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2976     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2977         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2978         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2979         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2980     } else {
2981         die "unkown audio device '$audio->{dev}', implement me!";
2982     }
2983
2984     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2985
2986     return $devs;
2987 }
2988
2989 sub get_tpm_paths {
2990     my ($vmid) = @_;
2991     return {
2992         socket => "/var/run/qemu-server/$vmid.swtpm",
2993         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
2994     };
2995 }
2996
2997 sub add_tpm_device {
2998     my ($vmid, $devices, $conf) = @_;
2999
3000     return if !$conf->{tpmstate0};
3001
3002     my $paths = get_tpm_paths($vmid);
3003
3004     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3005     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3006     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3007 }
3008
3009 sub start_swtpm {
3010     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3011
3012     return if !$tpmdrive;
3013
3014     my $state;
3015     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3016     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3017     if ($storeid) {
3018         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3019     } else {
3020         $state = $tpm->{file};
3021     }
3022
3023     my $paths = get_tpm_paths($vmid);
3024
3025     # during migration, we will get state from remote
3026     #
3027     if (!$migration) {
3028         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3029         my $setup_cmd = [
3030             "swtpm_setup",
3031             "--tpmstate",
3032             "file://$state",
3033             "--createek",
3034             "--create-ek-cert",
3035             "--create-platform-cert",
3036             "--lock-nvram",
3037             "--config",
3038             "/etc/swtpm_setup.conf", # do not use XDG configs
3039             "--runas",
3040             "0", # force creation as root, error if not possible
3041             "--not-overwrite", # ignore existing state, do not modify
3042         ];
3043
3044         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3045         # TPM 2.0 supports ECC crypto, use if possible
3046         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3047
3048         run_command($setup_cmd, outfunc => sub {
3049             print "swtpm_setup: $1\n";
3050         });
3051     }
3052
3053     my $emulator_cmd = [
3054         "swtpm",
3055         "socket",
3056         "--tpmstate",
3057         "backend-uri=file://$state,mode=0600",
3058         "--ctrl",
3059         "type=unixio,path=$paths->{socket},mode=0600",
3060         "--pid",
3061         "file=$paths->{pid}",
3062         "--terminate", # terminate on QEMU disconnect
3063         "--daemon",
3064     ];
3065     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3066     run_command($emulator_cmd, outfunc => sub { print $1; });
3067
3068     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3069     while (! -e $paths->{pid}) {
3070         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3071         usleep(50_000);
3072     }
3073
3074     # return untainted PID of swtpm daemon so it can be killed on error
3075     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3076     return $1;
3077 }
3078
3079 sub vga_conf_has_spice {
3080     my ($vga) = @_;
3081
3082     my $vgaconf = parse_vga($vga);
3083     my $vgatype = $vgaconf->{type};
3084     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3085
3086     return $1 || 1;
3087 }
3088
3089 sub is_native($) {
3090     my ($arch) = @_;
3091     return get_host_arch() eq $arch;
3092 }
3093
3094 sub get_vm_arch {
3095     my ($conf) = @_;
3096     return $conf->{arch} // get_host_arch();
3097 }
3098
3099 my $default_machines = {
3100     x86_64 => 'pc',
3101     aarch64 => 'virt',
3102 };
3103
3104 sub get_installed_machine_version {
3105     my ($kvmversion) = @_;
3106     $kvmversion = kvm_user_version() if !defined($kvmversion);
3107     $kvmversion =~ m/^(\d+\.\d+)/;
3108     return $1;
3109 }
3110
3111 sub windows_get_pinned_machine_version {
3112     my ($machine, $base_version, $kvmversion) = @_;
3113
3114     my $pin_version = $base_version;
3115     if (!defined($base_version) ||
3116         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3117     ) {
3118         $pin_version = get_installed_machine_version($kvmversion);
3119     }
3120     if (!$machine || $machine eq 'pc') {
3121         $machine = "pc-i440fx-$pin_version";
3122     } elsif ($machine eq 'q35') {
3123         $machine = "pc-q35-$pin_version";
3124     } elsif ($machine eq 'virt') {
3125         $machine = "virt-$pin_version";
3126     } else {
3127         warn "unknown machine type '$machine', not touching that!\n";
3128     }
3129
3130     return $machine;
3131 }
3132
3133 sub get_vm_machine {
3134     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3135
3136     my $machine = $forcemachine || $conf->{machine};
3137
3138     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3139         $kvmversion //= kvm_user_version();
3140         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3141         # layout which confuses windows quite a bit and may result in various regressions..
3142         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3143         if (windows_version($conf->{ostype})) {
3144             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3145         }
3146         $arch //= 'x86_64';
3147         $machine ||= $default_machines->{$arch};
3148         if ($add_pve_version) {
3149             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3150             $machine .= "+pve$pvever";
3151         }
3152     }
3153
3154     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3155         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3156         $machine = $1 if $is_pxe;
3157
3158         # for version-pinned machines that do not include a pve-version (e.g.
3159         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3160         $machine .= '+pve0';
3161
3162         $machine .= '.pxe' if $is_pxe;
3163     }
3164
3165     return $machine;
3166 }
3167
3168 sub get_ovmf_files($$) {
3169     my ($arch, $efidisk) = @_;
3170
3171     my $types = $OVMF->{$arch}
3172         or die "no OVMF images known for architecture '$arch'\n";
3173
3174     my $type = 'default';
3175     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3176         $type = $efidisk->{'pre-enrolled-keys'} ? "4m-ms" : "4m";
3177     }
3178
3179     return $types->{$type}->@*;
3180 }
3181
3182 my $Arch2Qemu = {
3183     aarch64 => '/usr/bin/qemu-system-aarch64',
3184     x86_64 => '/usr/bin/qemu-system-x86_64',
3185 };
3186 sub get_command_for_arch($) {
3187     my ($arch) = @_;
3188     return '/usr/bin/kvm' if is_native($arch);
3189
3190     my $cmd = $Arch2Qemu->{$arch}
3191         or die "don't know how to emulate architecture '$arch'\n";
3192     return $cmd;
3193 }
3194
3195 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3196 # to use in a QEMU command line (-cpu element), first array_intersect the result
3197 # of query_supported_ with query_understood_. This is necessary because:
3198 #
3199 # a) query_understood_ returns flags the host cannot use and
3200 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3201 #    flags, but CPU settings - with most of them being flags. Those settings
3202 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3203 #
3204 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3205 # expensive. If you need the value returned from this, you can get it much
3206 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3207 # $accel being 'kvm' or 'tcg'.
3208 #
3209 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3210 # changes, automatically populating pmxcfs.
3211 #
3212 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3213 # since kvm and tcg machines support different flags
3214 #
3215 sub query_supported_cpu_flags {
3216     my ($arch) = @_;
3217
3218     $arch //= get_host_arch();
3219     my $default_machine = $default_machines->{$arch};
3220
3221     my $flags = {};
3222
3223     # FIXME: Once this is merged, the code below should work for ARM as well:
3224     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3225     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3226         $arch eq "aarch64";
3227
3228     my $kvm_supported = defined(kvm_version());
3229     my $qemu_cmd = get_command_for_arch($arch);
3230     my $fakevmid = -1;
3231     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3232
3233     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3234     my $query_supported_run_qemu = sub {
3235         my ($kvm) = @_;
3236
3237         my $flags = {};
3238         my $cmd = [
3239             $qemu_cmd,
3240             '-machine', $default_machine,
3241             '-display', 'none',
3242             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3243             '-mon', 'chardev=qmp,mode=control',
3244             '-pidfile', $pidfile,
3245             '-S', '-daemonize'
3246         ];
3247
3248         if (!$kvm) {
3249             push @$cmd, '-accel', 'tcg';
3250         }
3251
3252         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3253         die "QEMU flag querying VM exited with code " . $rc if $rc;
3254
3255         eval {
3256             my $cmd_result = mon_cmd(
3257                 $fakevmid,
3258                 'query-cpu-model-expansion',
3259                 type => 'full',
3260                 model => { name => 'host' }
3261             );
3262
3263             my $props = $cmd_result->{model}->{props};
3264             foreach my $prop (keys %$props) {
3265                 next if $props->{$prop} ne '1';
3266                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3267                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3268                 # We only keep those with underscores, to match /proc/cpuinfo
3269                 $prop =~ s/\.|-/_/g;
3270                 $flags->{$prop} = 1;
3271             }
3272         };
3273         my $err = $@;
3274
3275         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3276         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3277
3278         die $err if $err;
3279
3280         return [ sort keys %$flags ];
3281     };
3282
3283     # We need to query QEMU twice, since KVM and TCG have different supported flags
3284     PVE::QemuConfig->lock_config($fakevmid, sub {
3285         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3286         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3287
3288         if ($kvm_supported) {
3289             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3290             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3291         }
3292     });
3293
3294     return $flags;
3295 }
3296
3297 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3298 my $understood_cpu_flag_dir = "/usr/share/kvm";
3299 sub query_understood_cpu_flags {
3300     my $arch = get_host_arch();
3301     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3302
3303     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3304         if ! -e $filepath;
3305
3306     my $raw = file_get_contents($filepath);
3307     $raw =~ s/^\s+|\s+$//g;
3308     my @flags = split(/\s+/, $raw);
3309
3310     return \@flags;
3311 }
3312
3313 my sub get_cpuunits {
3314     my ($conf) = @_;
3315     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3316 }
3317 sub config_to_command {
3318     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3319         $pbs_backing) = @_;
3320
3321     my $cmd = [];
3322     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3323     my $devices = [];
3324     my $bridges = {};
3325     my $ostype = $conf->{ostype};
3326     my $winversion = windows_version($ostype);
3327     my $kvm = $conf->{kvm};
3328     my $nodename = nodename();
3329
3330     my $arch = get_vm_arch($conf);
3331     my $kvm_binary = get_command_for_arch($arch);
3332     my $kvmver = kvm_user_version($kvm_binary);
3333
3334     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3335         $kvmver //= "undefined";
3336         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3337     }
3338
3339     my $add_pve_version = min_version($kvmver, 4, 1);
3340
3341     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3342     my $machine_version = extract_version($machine_type, $kvmver);
3343     $kvm //= 1 if is_native($arch);
3344
3345     $machine_version =~ m/(\d+)\.(\d+)/;
3346     my ($machine_major, $machine_minor) = ($1, $2);
3347
3348     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3349         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3350     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3351         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3352             ." please upgrade node '$nodename'\n"
3353     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3354         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3355         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3356             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3357             ." node '$nodename'\n";
3358     }
3359
3360     # if a specific +pve version is required for a feature, use $version_guard
3361     # instead of min_version to allow machines to be run with the minimum
3362     # required version
3363     my $required_pve_version = 0;
3364     my $version_guard = sub {
3365         my ($major, $minor, $pve) = @_;
3366         return 0 if !min_version($machine_version, $major, $minor, $pve);
3367         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3368         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3369         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3370         return 1;
3371     };
3372
3373     if ($kvm && !defined kvm_version()) {
3374         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3375             ." or enable in BIOS.\n";
3376     }
3377
3378     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3379     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3380     my $use_old_bios_files = undef;
3381     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3382
3383     my $cpuunits = get_cpuunits($conf);
3384
3385     push @$cmd, $kvm_binary;
3386
3387     push @$cmd, '-id', $vmid;
3388
3389     my $vmname = $conf->{name} || "vm$vmid";
3390
3391     push @$cmd, '-name', $vmname;
3392
3393     push @$cmd, '-no-shutdown';
3394
3395     my $use_virtio = 0;
3396
3397     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3398     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3399     push @$cmd, '-mon', "chardev=qmp,mode=control";
3400
3401     if (min_version($machine_version, 2, 12)) {
3402         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3403         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3404     }
3405
3406     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3407
3408     push @$cmd, '-daemonize';
3409
3410     if ($conf->{smbios1}) {
3411         my $smbios_conf = parse_smbios1($conf->{smbios1});
3412         if ($smbios_conf->{base64}) {
3413             # Do not pass base64 flag to qemu
3414             delete $smbios_conf->{base64};
3415             my $smbios_string = "";
3416             foreach my $key (keys %$smbios_conf) {
3417                 my $value;
3418                 if ($key eq "uuid") {
3419                     $value = $smbios_conf->{uuid}
3420                 } else {
3421                     $value = decode_base64($smbios_conf->{$key});
3422                 }
3423                 # qemu accepts any binary data, only commas need escaping by double comma
3424                 $value =~ s/,/,,/g;
3425                 $smbios_string .= "," . $key . "=" . $value if $value;
3426             }
3427             push @$cmd, '-smbios', "type=1" . $smbios_string;
3428         } else {
3429             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3430         }
3431     }
3432
3433     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3434         my $d;
3435         if (my $efidisk = $conf->{efidisk0}) {
3436             $d = parse_drive('efidisk0', $efidisk);
3437         }
3438
3439         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d);
3440         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3441
3442         my ($path, $format);
3443         my $read_only_str = '';
3444         if ($d) {
3445             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3446             $format = $d->{format};
3447             if ($storeid) {
3448                 $path = PVE::Storage::path($storecfg, $d->{file});
3449                 if (!defined($format)) {
3450                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3451                     $format = qemu_img_format($scfg, $volname);
3452                 }
3453             } else {
3454                 $path = $d->{file};
3455                 die "efidisk format must be specified\n"
3456                     if !defined($format);
3457             }
3458
3459             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3460         } else {
3461             warn "no efidisk configured! Using temporary efivars disk.\n";
3462             $path = "/tmp/$vmid-ovmf.fd";
3463             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3464             $format = 'raw';
3465         }
3466
3467         my $size_str = "";
3468
3469         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3470             $size_str = ",size=" . (-s $ovmf_vars);
3471         }
3472
3473         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3474         my $cache = "";
3475         if ($path =~ m/^rbd:/) {
3476                 $cache = ',cache=writeback';
3477                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3478         }
3479
3480         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3481         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3482     }
3483
3484     if ($q35) { # tell QEMU to load q35 config early
3485         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3486         if (min_version($machine_version, 4, 0)) {
3487             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3488         } else {
3489             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3490         }
3491     }
3492
3493     if ($conf->{vmgenid}) {
3494         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3495     }
3496
3497     # add usb controllers
3498     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3499         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3500     push @$devices, @usbcontrollers if @usbcontrollers;
3501     my $vga = parse_vga($conf->{vga});
3502
3503     my $qxlnum = vga_conf_has_spice($conf->{vga});
3504     $vga->{type} = 'qxl' if $qxlnum;
3505
3506     if (!$vga->{type}) {
3507         if ($arch eq 'aarch64') {
3508             $vga->{type} = 'virtio';
3509         } elsif (min_version($machine_version, 2, 9)) {
3510             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3511         } else {
3512             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3513         }
3514     }
3515
3516     # enable absolute mouse coordinates (needed by vnc)
3517     my $tablet = $conf->{tablet};
3518     if (!defined($tablet)) {
3519         $tablet = $defaults->{tablet};
3520         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3521         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3522     }
3523
3524     if ($tablet) {
3525         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3526         my $kbd = print_keyboarddevice_full($conf, $arch);
3527         push @$devices, '-device', $kbd if defined($kbd);
3528     }
3529
3530     my $bootorder = device_bootorder($conf);
3531
3532     # host pci device passthrough
3533     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3534         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3535
3536     # usb devices
3537     my $usb_dev_features = {};
3538     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3539
3540     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3541         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3542     push @$devices, @usbdevices if @usbdevices;
3543
3544     # serial devices
3545     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3546         my $path = $conf->{"serial$i"} or next;
3547         if ($path eq 'socket') {
3548             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3549             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3550             # On aarch64, serial0 is the UART device. Qemu only allows
3551             # connecting UART devices via the '-serial' command line, as
3552             # the device has a fixed slot on the hardware...
3553             if ($arch eq 'aarch64' && $i == 0) {
3554                 push @$devices, '-serial', "chardev:serial$i";
3555             } else {
3556                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3557             }
3558         } else {
3559             die "no such serial device\n" if ! -c $path;
3560             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3561             push @$devices, '-device', "isa-serial,chardev=serial$i";
3562         }
3563     }
3564
3565     # parallel devices
3566     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3567         if (my $path = $conf->{"parallel$i"}) {
3568             die "no such parallel device\n" if ! -c $path;
3569             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3570             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3571             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3572         }
3573     }
3574
3575     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3576         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3577         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3578         push @$devices, @$audio_devs;
3579     }
3580
3581     add_tpm_device($vmid, $devices, $conf);
3582
3583     my $sockets = 1;
3584     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3585     $sockets = $conf->{sockets} if  $conf->{sockets};
3586
3587     my $cores = $conf->{cores} || 1;
3588
3589     my $maxcpus = $sockets * $cores;
3590
3591     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3592
3593     my $allowed_vcpus = $cpuinfo->{cpus};
3594
3595     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3596
3597     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3598         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3599         for (my $i = 2; $i <= $vcpus; $i++)  {
3600             my $cpustr = print_cpu_device($conf,$i);
3601             push @$cmd, '-device', $cpustr;
3602         }
3603
3604     } else {
3605
3606         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3607     }
3608     push @$cmd, '-nodefaults';
3609
3610     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3611
3612     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3613
3614     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3615
3616     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3617         push @$devices, '-device', print_vga_device(
3618             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3619         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3620         push @$cmd,  '-vnc', "unix:$socket,password=on";
3621     } else {
3622         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3623         push @$cmd, '-nographic';
3624     }
3625
3626     # time drift fix
3627     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3628     my $useLocaltime = $conf->{localtime};
3629
3630     if ($winversion >= 5) { # windows
3631         $useLocaltime = 1 if !defined($conf->{localtime});
3632
3633         # use time drift fix when acpi is enabled
3634         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3635             $tdf = 1 if !defined($conf->{tdf});
3636         }
3637     }
3638
3639     if ($winversion >= 6) {
3640         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3641         push @$cmd, '-no-hpet';
3642     }
3643
3644     push @$rtcFlags, 'driftfix=slew' if $tdf;
3645
3646     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3647         push @$rtcFlags, "base=$conf->{startdate}";
3648     } elsif ($useLocaltime) {
3649         push @$rtcFlags, 'base=localtime';
3650     }
3651
3652     if ($forcecpu) {
3653         push @$cmd, '-cpu', $forcecpu;
3654     } else {
3655         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3656     }
3657
3658     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3659
3660     push @$cmd, '-S' if $conf->{freeze};
3661
3662     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3663
3664     my $guest_agent = parse_guest_agent($conf);
3665
3666     if ($guest_agent->{enabled}) {
3667         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3668         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3669
3670         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3671             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3672             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3673             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3674         } elsif ($guest_agent->{type} eq 'isa') {
3675             push @$devices, '-device', "isa-serial,chardev=qga0";
3676         }
3677     }
3678
3679     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3680     if ($rng && $version_guard->(4, 1, 2)) {
3681         check_rng_source($rng->{source});
3682
3683         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3684         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3685         my $limiter_str = "";
3686         if ($max_bytes) {
3687             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3688         }
3689
3690         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3691         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3692         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3693     }
3694
3695     my $spice_port;
3696
3697     if ($qxlnum) {
3698         if ($qxlnum > 1) {
3699             if ($winversion){
3700                 for (my $i = 1; $i < $qxlnum; $i++){
3701                     push @$devices, '-device', print_vga_device(
3702                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3703                 }
3704             } else {
3705                 # assume other OS works like Linux
3706                 my ($ram, $vram) = ("134217728", "67108864");
3707                 if ($vga->{memory}) {
3708                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3709                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3710                 }
3711                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3712                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3713             }
3714         }
3715
3716         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3717
3718         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3719         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3720         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3721
3722         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3723         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3724         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3725
3726         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3727         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3728
3729         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3730         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3731         if ($spice_enhancement->{foldersharing}) {
3732             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3733             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3734         }
3735
3736         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3737         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3738             if $spice_enhancement->{videostreaming};
3739
3740         push @$devices, '-spice', "$spice_opts";
3741     }
3742
3743     # enable balloon by default, unless explicitly disabled
3744     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3745         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3746         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3747     }
3748
3749     if ($conf->{watchdog}) {
3750         my $wdopts = parse_watchdog($conf->{watchdog});
3751         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3752         my $watchdog = $wdopts->{model} || 'i6300esb';
3753         push @$devices, '-device', "$watchdog$pciaddr";
3754         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3755     }
3756
3757     my $vollist = [];
3758     my $scsicontroller = {};
3759     my $ahcicontroller = {};
3760     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3761
3762     # Add iscsi initiator name if available
3763     if (my $initiator = get_initiator_name()) {
3764         push @$devices, '-iscsi', "initiator-name=$initiator";
3765     }
3766
3767     PVE::QemuConfig->foreach_volume($conf, sub {
3768         my ($ds, $drive) = @_;
3769
3770         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3771             check_volume_storage_type($storecfg, $drive->{file});
3772             push @$vollist, $drive->{file};
3773         }
3774
3775         # ignore efidisk here, already added in bios/fw handling code above
3776         return if $drive->{interface} eq 'efidisk';
3777         # similar for TPM
3778         return if $drive->{interface} eq 'tpmstate';
3779
3780         $use_virtio = 1 if $ds =~ m/^virtio/;
3781
3782         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3783
3784         if ($drive->{interface} eq 'virtio'){
3785            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3786         }
3787
3788         if ($drive->{interface} eq 'scsi') {
3789
3790             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3791
3792             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3793                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3794
3795             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3796             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3797
3798             my $iothread = '';
3799             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3800                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3801                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3802             } elsif ($drive->{iothread}) {
3803                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3804             }
3805
3806             my $queues = '';
3807             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3808                 $queues = ",num_queues=$drive->{queues}";
3809             }
3810
3811             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3812                 if !$scsicontroller->{$controller};
3813             $scsicontroller->{$controller}=1;
3814         }
3815
3816         if ($drive->{interface} eq 'sata') {
3817             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3818             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3819             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3820                 if !$ahcicontroller->{$controller};
3821             $ahcicontroller->{$controller}=1;
3822         }
3823
3824         my $pbs_conf = $pbs_backing->{$ds};
3825         my $pbs_name = undef;
3826         if ($pbs_conf) {
3827             $pbs_name = "drive-$ds-pbs";
3828             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3829         }
3830
3831         my $drive_cmd = print_drive_commandline_full(
3832             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3833
3834         # extra protection for templates, but SATA and IDE don't support it..
3835         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3836
3837         push @$devices, '-drive',$drive_cmd;
3838         push @$devices, '-device', print_drivedevice_full(
3839             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3840     });
3841
3842     for (my $i = 0; $i < $MAX_NETS; $i++) {
3843         my $netname = "net$i";
3844
3845         next if !$conf->{$netname};
3846         my $d = parse_net($conf->{$netname});
3847         next if !$d;
3848
3849         $use_virtio = 1 if $d->{model} eq 'virtio';
3850
3851         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3852
3853         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3854         push @$devices, '-netdev', $netdevfull;
3855
3856         my $netdevicefull = print_netdevice_full(
3857             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3858
3859         push @$devices, '-device', $netdevicefull;
3860     }
3861
3862     if ($conf->{ivshmem}) {
3863         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3864
3865         my $bus;
3866         if ($q35) {
3867             $bus = print_pcie_addr("ivshmem");
3868         } else {
3869             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3870         }
3871
3872         my $ivshmem_name = $ivshmem->{name} // $vmid;
3873         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3874
3875         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3876         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3877             .",size=$ivshmem->{size}M";
3878     }
3879
3880     # pci.4 is nested in pci.1
3881     $bridges->{1} = 1 if $bridges->{4};
3882
3883     if (!$q35) { # add pci bridges
3884         if (min_version($machine_version, 2, 3)) {
3885            $bridges->{1} = 1;
3886            $bridges->{2} = 1;
3887         }
3888         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3889     }
3890
3891     for my $k (sort {$b cmp $a} keys %$bridges) {
3892         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3893
3894         my $k_name = $k;
3895         if ($k == 2 && $legacy_igd) {
3896             $k_name = "$k-igd";
3897         }
3898         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3899         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3900
3901         if ($q35) { # add after -readconfig pve-q35.cfg
3902             splice @$devices, 2, 0, '-device', $devstr;
3903         } else {
3904             unshift @$devices, '-device', $devstr if $k > 0;
3905         }
3906     }
3907
3908     if (!$kvm) {
3909         push @$machineFlags, 'accel=tcg';
3910     }
3911
3912     my $machine_type_min = $machine_type;
3913     if ($add_pve_version) {
3914         $machine_type_min =~ s/\+pve\d+$//;
3915         $machine_type_min .= "+pve$required_pve_version";
3916     }
3917     push @$machineFlags, "type=${machine_type_min}";
3918
3919     push @$cmd, @$devices;
3920     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3921     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3922     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3923
3924     if (my $vmstate = $conf->{vmstate}) {
3925         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3926         push @$vollist, $vmstate;
3927         push @$cmd, '-loadstate', $statepath;
3928         print "activating and using '$vmstate' as vmstate\n";
3929     }
3930
3931     if (PVE::QemuConfig->is_template($conf)) {
3932         # needed to workaround base volumes being read-only
3933         push @$cmd, '-snapshot';
3934     }
3935
3936     # add custom args
3937     if ($conf->{args}) {
3938         my $aa = PVE::Tools::split_args($conf->{args});
3939         push @$cmd, @$aa;
3940     }
3941
3942     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3943 }
3944
3945 sub check_rng_source {
3946     my ($source) = @_;
3947
3948     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3949     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3950         if ! -e $source;
3951
3952     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3953     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3954         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3955         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3956         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3957             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3958             ." to the host.\n";
3959     }
3960 }
3961
3962 sub spice_port {
3963     my ($vmid) = @_;
3964
3965     my $res = mon_cmd($vmid, 'query-spice');
3966
3967     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3968 }
3969
3970 sub vm_devices_list {
3971     my ($vmid) = @_;
3972
3973     my $res = mon_cmd($vmid, 'query-pci');
3974     my $devices_to_check = [];
3975     my $devices = {};
3976     foreach my $pcibus (@$res) {
3977         push @$devices_to_check, @{$pcibus->{devices}},
3978     }
3979
3980     while (@$devices_to_check) {
3981         my $to_check = [];
3982         for my $d (@$devices_to_check) {
3983             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3984             next if !$d->{'pci_bridge'};
3985
3986             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3987             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3988         }
3989         $devices_to_check = $to_check;
3990     }
3991
3992     my $resblock = mon_cmd($vmid, 'query-block');
3993     foreach my $block (@$resblock) {
3994         if($block->{device} =~ m/^drive-(\S+)/){
3995                 $devices->{$1} = 1;
3996         }
3997     }
3998
3999     my $resmice = mon_cmd($vmid, 'query-mice');
4000     foreach my $mice (@$resmice) {
4001         if ($mice->{name} eq 'QEMU HID Tablet') {
4002             $devices->{tablet} = 1;
4003             last;
4004         }
4005     }
4006
4007     # for usb devices there is no query-usb
4008     # but we can iterate over the entries in
4009     # qom-list path=/machine/peripheral
4010     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4011     foreach my $per (@$resperipheral) {
4012         if ($per->{name} =~ m/^usb\d+$/) {
4013             $devices->{$per->{name}} = 1;
4014         }
4015     }
4016
4017     return $devices;
4018 }
4019
4020 sub vm_deviceplug {
4021     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4022
4023     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4024
4025     my $devices_list = vm_devices_list($vmid);
4026     return 1 if defined($devices_list->{$deviceid});
4027
4028     # add PCI bridge if we need it for the device
4029     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4030
4031     if ($deviceid eq 'tablet') {
4032         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4033     } elsif ($deviceid eq 'keyboard') {
4034         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4035     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4036         die "usb hotplug currently not reliable\n";
4037         # since we can't reliably hot unplug all added usb devices and usb
4038         # passthrough breaks live migration we disable usb hotplugging for now
4039         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4040     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4041         qemu_iothread_add($vmid, $deviceid, $device);
4042
4043         qemu_driveadd($storecfg, $vmid, $device);
4044         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4045
4046         qemu_deviceadd($vmid, $devicefull);
4047         eval { qemu_deviceaddverify($vmid, $deviceid); };
4048         if (my $err = $@) {
4049             eval { qemu_drivedel($vmid, $deviceid); };
4050             warn $@ if $@;
4051             die $err;
4052         }
4053     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4054         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4055         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4056         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4057
4058         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4059
4060         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4061             qemu_iothread_add($vmid, $deviceid, $device);
4062             $devicefull .= ",iothread=iothread-$deviceid";
4063         }
4064
4065         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4066             $devicefull .= ",num_queues=$device->{queues}";
4067         }
4068
4069         qemu_deviceadd($vmid, $devicefull);
4070         qemu_deviceaddverify($vmid, $deviceid);
4071     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4072         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4073         qemu_driveadd($storecfg, $vmid, $device);
4074
4075         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4076         eval { qemu_deviceadd($vmid, $devicefull); };
4077         if (my $err = $@) {
4078             eval { qemu_drivedel($vmid, $deviceid); };
4079             warn $@ if $@;
4080             die $err;
4081         }
4082     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4083         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4084
4085         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4086         my $use_old_bios_files = undef;
4087         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4088
4089         my $netdevicefull = print_netdevice_full(
4090             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4091         qemu_deviceadd($vmid, $netdevicefull);
4092         eval {
4093             qemu_deviceaddverify($vmid, $deviceid);
4094             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4095         };
4096         if (my $err = $@) {
4097             eval { qemu_netdevdel($vmid, $deviceid); };
4098             warn $@ if $@;
4099             die $err;
4100         }
4101     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4102         my $bridgeid = $2;
4103         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4104         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4105
4106         qemu_deviceadd($vmid, $devicefull);
4107         qemu_deviceaddverify($vmid, $deviceid);
4108     } else {
4109         die "can't hotplug device '$deviceid'\n";
4110     }
4111
4112     return 1;
4113 }
4114
4115 # fixme: this should raise exceptions on error!
4116 sub vm_deviceunplug {
4117     my ($vmid, $conf, $deviceid) = @_;
4118
4119     my $devices_list = vm_devices_list($vmid);
4120     return 1 if !defined($devices_list->{$deviceid});
4121
4122     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4123     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4124
4125     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4126         qemu_devicedel($vmid, $deviceid);
4127     } elsif ($deviceid =~ m/^usb\d+$/) {
4128         die "usb hotplug currently not reliable\n";
4129         # when unplugging usb devices this way, there may be remaining usb
4130         # controllers/hubs so we disable it for now
4131         #qemu_devicedel($vmid, $deviceid);
4132         #qemu_devicedelverify($vmid, $deviceid);
4133     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4134         my $device = parse_drive($deviceid, $conf->{$deviceid});
4135
4136         qemu_devicedel($vmid, $deviceid);
4137         qemu_devicedelverify($vmid, $deviceid);
4138         qemu_drivedel($vmid, $deviceid);
4139         qemu_iothread_del($vmid, $deviceid, $device);
4140     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4141         qemu_devicedel($vmid, $deviceid);
4142         qemu_devicedelverify($vmid, $deviceid);
4143     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4144         my $device = parse_drive($deviceid, $conf->{$deviceid});
4145
4146         qemu_devicedel($vmid, $deviceid);
4147         qemu_drivedel($vmid, $deviceid);
4148         qemu_deletescsihw($conf, $vmid, $deviceid);
4149
4150         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4151             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4152     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4153         qemu_devicedel($vmid, $deviceid);
4154         qemu_devicedelverify($vmid, $deviceid);
4155         qemu_netdevdel($vmid, $deviceid);
4156     } else {
4157         die "can't unplug device '$deviceid'\n";
4158     }
4159
4160     return 1;
4161 }
4162
4163 sub qemu_deviceadd {
4164     my ($vmid, $devicefull) = @_;
4165
4166     $devicefull = "driver=".$devicefull;
4167     my %options =  split(/[=,]/, $devicefull);
4168
4169     mon_cmd($vmid, "device_add" , %options);
4170 }
4171
4172 sub qemu_devicedel {
4173     my ($vmid, $deviceid) = @_;
4174
4175     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4176 }
4177
4178 sub qemu_iothread_add {
4179     my ($vmid, $deviceid, $device) = @_;
4180
4181     if ($device->{iothread}) {
4182         my $iothreads = vm_iothreads_list($vmid);
4183         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4184     }
4185 }
4186
4187 sub qemu_iothread_del {
4188     my ($vmid, $deviceid, $device) = @_;
4189
4190     if ($device->{iothread}) {
4191         my $iothreads = vm_iothreads_list($vmid);
4192         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4193     }
4194 }
4195
4196 sub qemu_objectadd {
4197     my ($vmid, $objectid, $qomtype) = @_;
4198
4199     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4200
4201     return 1;
4202 }
4203
4204 sub qemu_objectdel {
4205     my ($vmid, $objectid) = @_;
4206
4207     mon_cmd($vmid, "object-del", id => $objectid);
4208
4209     return 1;
4210 }
4211
4212 sub qemu_driveadd {
4213     my ($storecfg, $vmid, $device) = @_;
4214
4215     my $kvmver = get_running_qemu_version($vmid);
4216     my $io_uring = min_version($kvmver, 6, 0);
4217     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4218     $drive =~ s/\\/\\\\/g;
4219     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4220
4221     # If the command succeeds qemu prints: "OK"
4222     return 1 if $ret =~ m/OK/s;
4223
4224     die "adding drive failed: $ret\n";
4225 }
4226
4227 sub qemu_drivedel {
4228     my ($vmid, $deviceid) = @_;
4229
4230     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4231     $ret =~ s/^\s+//;
4232
4233     return 1 if $ret eq "";
4234
4235     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4236     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4237
4238     die "deleting drive $deviceid failed : $ret\n";
4239 }
4240
4241 sub qemu_deviceaddverify {
4242     my ($vmid, $deviceid) = @_;
4243
4244     for (my $i = 0; $i <= 5; $i++) {
4245          my $devices_list = vm_devices_list($vmid);
4246          return 1 if defined($devices_list->{$deviceid});
4247          sleep 1;
4248     }
4249
4250     die "error on hotplug device '$deviceid'\n";
4251 }
4252
4253
4254 sub qemu_devicedelverify {
4255     my ($vmid, $deviceid) = @_;
4256
4257     # need to verify that the device is correctly removed as device_del
4258     # is async and empty return is not reliable
4259
4260     for (my $i = 0; $i <= 5; $i++) {
4261          my $devices_list = vm_devices_list($vmid);
4262          return 1 if !defined($devices_list->{$deviceid});
4263          sleep 1;
4264     }
4265
4266     die "error on hot-unplugging device '$deviceid'\n";
4267 }
4268
4269 sub qemu_findorcreatescsihw {
4270     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4271
4272     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4273
4274     my $scsihwid="$controller_prefix$controller";
4275     my $devices_list = vm_devices_list($vmid);
4276
4277     if (!defined($devices_list->{$scsihwid})) {
4278         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4279     }
4280
4281     return 1;
4282 }
4283
4284 sub qemu_deletescsihw {
4285     my ($conf, $vmid, $opt) = @_;
4286
4287     my $device = parse_drive($opt, $conf->{$opt});
4288
4289     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4290         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4291         return 1;
4292     }
4293
4294     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4295
4296     my $devices_list = vm_devices_list($vmid);
4297     foreach my $opt (keys %{$devices_list}) {
4298         if (is_valid_drivename($opt)) {
4299             my $drive = parse_drive($opt, $conf->{$opt});
4300             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4301                 return 1;
4302             }
4303         }
4304     }
4305
4306     my $scsihwid="scsihw$controller";
4307
4308     vm_deviceunplug($vmid, $conf, $scsihwid);
4309
4310     return 1;
4311 }
4312
4313 sub qemu_add_pci_bridge {
4314     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4315
4316     my $bridges = {};
4317
4318     my $bridgeid;
4319
4320     print_pci_addr($device, $bridges, $arch, $machine_type);
4321
4322     while (my ($k, $v) = each %$bridges) {
4323         $bridgeid = $k;
4324     }
4325     return 1 if !defined($bridgeid) || $bridgeid < 1;
4326
4327     my $bridge = "pci.$bridgeid";
4328     my $devices_list = vm_devices_list($vmid);
4329
4330     if (!defined($devices_list->{$bridge})) {
4331         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4332     }
4333
4334     return 1;
4335 }
4336
4337 sub qemu_set_link_status {
4338     my ($vmid, $device, $up) = @_;
4339
4340     mon_cmd($vmid, "set_link", name => $device,
4341                up => $up ? JSON::true : JSON::false);
4342 }
4343
4344 sub qemu_netdevadd {
4345     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4346
4347     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4348     my %options =  split(/[=,]/, $netdev);
4349
4350     if (defined(my $vhost = $options{vhost})) {
4351         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4352     }
4353
4354     if (defined(my $queues = $options{queues})) {
4355         $options{queues} = $queues + 0;
4356     }
4357
4358     mon_cmd($vmid, "netdev_add",  %options);
4359     return 1;
4360 }
4361
4362 sub qemu_netdevdel {
4363     my ($vmid, $deviceid) = @_;
4364
4365     mon_cmd($vmid, "netdev_del", id => $deviceid);
4366 }
4367
4368 sub qemu_usb_hotplug {
4369     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4370
4371     return if !$device;
4372
4373     # remove the old one first
4374     vm_deviceunplug($vmid, $conf, $deviceid);
4375
4376     # check if xhci controller is necessary and available
4377     if ($device->{usb3}) {
4378
4379         my $devicelist = vm_devices_list($vmid);
4380
4381         if (!$devicelist->{xhci}) {
4382             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4383             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4384         }
4385     }
4386     my $d = parse_usb_device($device->{host});
4387     $d->{usb3} = $device->{usb3};
4388
4389     # add the new one
4390     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4391 }
4392
4393 sub qemu_cpu_hotplug {
4394     my ($vmid, $conf, $vcpus) = @_;
4395
4396     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4397
4398     my $sockets = 1;
4399     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4400     $sockets = $conf->{sockets} if  $conf->{sockets};
4401     my $cores = $conf->{cores} || 1;
4402     my $maxcpus = $sockets * $cores;
4403
4404     $vcpus = $maxcpus if !$vcpus;
4405
4406     die "you can't add more vcpus than maxcpus\n"
4407         if $vcpus > $maxcpus;
4408
4409     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4410
4411     if ($vcpus < $currentvcpus) {
4412
4413         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4414
4415             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4416                 qemu_devicedel($vmid, "cpu$i");
4417                 my $retry = 0;
4418                 my $currentrunningvcpus = undef;
4419                 while (1) {
4420                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4421                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4422                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4423                     $retry++;
4424                     sleep 1;
4425                 }
4426                 #update conf after each succesfull cpu unplug
4427                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4428                 PVE::QemuConfig->write_config($vmid, $conf);
4429             }
4430         } else {
4431             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4432         }
4433
4434         return;
4435     }
4436
4437     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4438     die "vcpus in running vm does not match its configuration\n"
4439         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4440
4441     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4442
4443         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4444             my $cpustr = print_cpu_device($conf, $i);
4445             qemu_deviceadd($vmid, $cpustr);
4446
4447             my $retry = 0;
4448             my $currentrunningvcpus = undef;
4449             while (1) {
4450                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4451                 last if scalar(@{$currentrunningvcpus}) == $i;
4452                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4453                 sleep 1;
4454                 $retry++;
4455             }
4456             #update conf after each succesfull cpu hotplug
4457             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4458             PVE::QemuConfig->write_config($vmid, $conf);
4459         }
4460     } else {
4461
4462         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4463             mon_cmd($vmid, "cpu-add", id => int($i));
4464         }
4465     }
4466 }
4467
4468 sub qemu_block_set_io_throttle {
4469     my ($vmid, $deviceid,
4470         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4471         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4472         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4473         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4474
4475     return if !check_running($vmid) ;
4476
4477     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4478         bps => int($bps),
4479         bps_rd => int($bps_rd),
4480         bps_wr => int($bps_wr),
4481         iops => int($iops),
4482         iops_rd => int($iops_rd),
4483         iops_wr => int($iops_wr),
4484         bps_max => int($bps_max),
4485         bps_rd_max => int($bps_rd_max),
4486         bps_wr_max => int($bps_wr_max),
4487         iops_max => int($iops_max),
4488         iops_rd_max => int($iops_rd_max),
4489         iops_wr_max => int($iops_wr_max),
4490         bps_max_length => int($bps_max_length),
4491         bps_rd_max_length => int($bps_rd_max_length),
4492         bps_wr_max_length => int($bps_wr_max_length),
4493         iops_max_length => int($iops_max_length),
4494         iops_rd_max_length => int($iops_rd_max_length),
4495         iops_wr_max_length => int($iops_wr_max_length),
4496     );
4497
4498 }
4499
4500 sub qemu_block_resize {
4501     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4502
4503     my $running = check_running($vmid);
4504
4505     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4506
4507     return if !$running;
4508
4509     my $padding = (1024 - $size % 1024) % 1024;
4510     $size = $size + $padding;
4511
4512     mon_cmd(
4513         $vmid,
4514         "block_resize",
4515         device => $deviceid,
4516         size => int($size),
4517         timeout => 60,
4518     );
4519 }
4520
4521 sub qemu_volume_snapshot {
4522     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4523
4524     my $running = check_running($vmid);
4525
4526     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4527         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4528     } else {
4529         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4530     }
4531 }
4532
4533 sub qemu_volume_snapshot_delete {
4534     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4535
4536     my $running = check_running($vmid);
4537
4538     if($running) {
4539
4540         $running = undef;
4541         my $conf = PVE::QemuConfig->load_config($vmid);
4542         PVE::QemuConfig->foreach_volume($conf, sub {
4543             my ($ds, $drive) = @_;
4544             $running = 1 if $drive->{file} eq $volid;
4545         });
4546     }
4547
4548     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4549         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4550     } else {
4551         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4552     }
4553 }
4554
4555 sub set_migration_caps {
4556     my ($vmid, $savevm) = @_;
4557
4558     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4559
4560     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4561     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4562
4563     my $cap_ref = [];
4564
4565     my $enabled_cap = {
4566         "auto-converge" => 1,
4567         "xbzrle" => 1,
4568         "x-rdma-pin-all" => 0,
4569         "zero-blocks" => 0,
4570         "compress" => 0,
4571         "dirty-bitmaps" => $dirty_bitmaps,
4572     };
4573
4574     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4575
4576     for my $supported_capability (@$supported_capabilities) {
4577         push @$cap_ref, {
4578             capability => $supported_capability->{capability},
4579             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4580         };
4581     }
4582
4583     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4584 }
4585
4586 sub foreach_volid {
4587     my ($conf, $func, @param) = @_;
4588
4589     my $volhash = {};
4590
4591     my $test_volid = sub {
4592         my ($key, $drive, $snapname) = @_;
4593
4594         my $volid = $drive->{file};
4595         return if !$volid;
4596
4597         $volhash->{$volid}->{cdrom} //= 1;
4598         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4599
4600         my $replicate = $drive->{replicate} // 1;
4601         $volhash->{$volid}->{replicate} //= 0;
4602         $volhash->{$volid}->{replicate} = 1 if $replicate;
4603
4604         $volhash->{$volid}->{shared} //= 0;
4605         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4606
4607         $volhash->{$volid}->{referenced_in_config} //= 0;
4608         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4609
4610         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4611             if defined($snapname);
4612
4613         my $size = $drive->{size};
4614         $volhash->{$volid}->{size} //= $size if $size;
4615
4616         $volhash->{$volid}->{is_vmstate} //= 0;
4617         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4618
4619         $volhash->{$volid}->{is_tpmstate} //= 0;
4620         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4621
4622         $volhash->{$volid}->{is_unused} //= 0;
4623         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4624
4625         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4626     };
4627
4628     my $include_opts = {
4629         extra_keys => ['vmstate'],
4630         include_unused => 1,
4631     };
4632
4633     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4634     foreach my $snapname (keys %{$conf->{snapshots}}) {
4635         my $snap = $conf->{snapshots}->{$snapname};
4636         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4637     }
4638
4639     foreach my $volid (keys %$volhash) {
4640         &$func($volid, $volhash->{$volid}, @param);
4641     }
4642 }
4643
4644 my $fast_plug_option = {
4645     'lock' => 1,
4646     'name' => 1,
4647     'onboot' => 1,
4648     'shares' => 1,
4649     'startup' => 1,
4650     'description' => 1,
4651     'protection' => 1,
4652     'vmstatestorage' => 1,
4653     'hookscript' => 1,
4654     'tags' => 1,
4655 };
4656
4657 # hotplug changes in [PENDING]
4658 # $selection hash can be used to only apply specified options, for
4659 # example: { cores => 1 } (only apply changed 'cores')
4660 # $errors ref is used to return error messages
4661 sub vmconfig_hotplug_pending {
4662     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4663
4664     my $defaults = load_defaults();
4665     my $arch = get_vm_arch($conf);
4666     my $machine_type = get_vm_machine($conf, undef, $arch);
4667
4668     # commit values which do not have any impact on running VM first
4669     # Note: those option cannot raise errors, we we do not care about
4670     # $selection and always apply them.
4671
4672     my $add_error = sub {
4673         my ($opt, $msg) = @_;
4674         $errors->{$opt} = "hotplug problem - $msg";
4675     };
4676
4677     my $changes = 0;
4678     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4679         if ($fast_plug_option->{$opt}) {
4680             $conf->{$opt} = $conf->{pending}->{$opt};
4681             delete $conf->{pending}->{$opt};
4682             $changes = 1;
4683         }
4684     }
4685
4686     if ($changes) {
4687         PVE::QemuConfig->write_config($vmid, $conf);
4688     }
4689
4690     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4691
4692     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4693     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4694     foreach my $opt (sort keys %$pending_delete_hash) {
4695         next if $selection && !$selection->{$opt};
4696         my $force = $pending_delete_hash->{$opt}->{force};
4697         eval {
4698             if ($opt eq 'hotplug') {
4699                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4700             } elsif ($opt eq 'tablet') {
4701                 die "skip\n" if !$hotplug_features->{usb};
4702                 if ($defaults->{tablet}) {
4703                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4704                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4705                         if $arch eq 'aarch64';
4706                 } else {
4707                     vm_deviceunplug($vmid, $conf, 'tablet');
4708                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4709                 }
4710             } elsif ($opt =~ m/^usb\d+/) {
4711                 die "skip\n";
4712                 # since we cannot reliably hot unplug usb devices we are disabling it
4713                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4714                 #vm_deviceunplug($vmid, $conf, $opt);
4715             } elsif ($opt eq 'vcpus') {
4716                 die "skip\n" if !$hotplug_features->{cpu};
4717                 qemu_cpu_hotplug($vmid, $conf, undef);
4718             } elsif ($opt eq 'balloon') {
4719                 # enable balloon device is not hotpluggable
4720                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4721                 # here we reset the ballooning value to memory
4722                 my $balloon = $conf->{memory} || $defaults->{memory};
4723                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4724             } elsif ($fast_plug_option->{$opt}) {
4725                 # do nothing
4726             } elsif ($opt =~ m/^net(\d+)$/) {
4727                 die "skip\n" if !$hotplug_features->{network};
4728                 vm_deviceunplug($vmid, $conf, $opt);
4729             } elsif (is_valid_drivename($opt)) {
4730                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4731                 vm_deviceunplug($vmid, $conf, $opt);
4732                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4733             } elsif ($opt =~ m/^memory$/) {
4734                 die "skip\n" if !$hotplug_features->{memory};
4735                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4736             } elsif ($opt eq 'cpuunits') {
4737                 $cgroup->change_cpu_shares(undef, 1024);
4738             } elsif ($opt eq 'cpulimit') {
4739                 $cgroup->change_cpu_quota(-1, 100000);
4740             } else {
4741                 die "skip\n";
4742             }
4743         };
4744         if (my $err = $@) {
4745             &$add_error($opt, $err) if $err ne "skip\n";
4746         } else {
4747             delete $conf->{$opt};
4748             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4749         }
4750     }
4751
4752     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4753     $apply_pending_cloudinit = sub {
4754         return if $apply_pending_cloudinit_done; # once is enough
4755         $apply_pending_cloudinit_done = 1; # once is enough
4756
4757         my ($key, $value) = @_;
4758
4759         my @cloudinit_opts = keys %$confdesc_cloudinit;
4760         foreach my $opt (keys %{$conf->{pending}}) {
4761             next if !grep { $_ eq $opt } @cloudinit_opts;
4762             $conf->{$opt} = delete $conf->{pending}->{$opt};
4763         }
4764
4765         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4766         foreach my $opt (sort keys %$pending_delete_hash) {
4767             next if !grep { $_ eq $opt } @cloudinit_opts;
4768             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4769             delete $conf->{$opt};
4770         }
4771
4772         my $new_conf = { %$conf };
4773         $new_conf->{$key} = $value;
4774         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4775     };
4776
4777     foreach my $opt (keys %{$conf->{pending}}) {
4778         next if $selection && !$selection->{$opt};
4779         my $value = $conf->{pending}->{$opt};
4780         eval {
4781             if ($opt eq 'hotplug') {
4782                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4783             } elsif ($opt eq 'tablet') {
4784                 die "skip\n" if !$hotplug_features->{usb};
4785                 if ($value == 1) {
4786                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4787                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4788                         if $arch eq 'aarch64';
4789                 } elsif ($value == 0) {
4790                     vm_deviceunplug($vmid, $conf, 'tablet');
4791                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4792                 }
4793             } elsif ($opt =~ m/^usb\d+$/) {
4794                 die "skip\n";
4795                 # since we cannot reliably hot unplug usb devices we disable it for now
4796                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4797                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4798                 #die "skip\n" if !$d;
4799                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4800             } elsif ($opt eq 'vcpus') {
4801                 die "skip\n" if !$hotplug_features->{cpu};
4802                 qemu_cpu_hotplug($vmid, $conf, $value);
4803             } elsif ($opt eq 'balloon') {
4804                 # enable/disable balloning device is not hotpluggable
4805                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4806                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4807                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4808
4809                 # allow manual ballooning if shares is set to zero
4810                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4811                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4812                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4813                 }
4814             } elsif ($opt =~ m/^net(\d+)$/) {
4815                 # some changes can be done without hotplug
4816                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4817                                     $vmid, $opt, $value, $arch, $machine_type);
4818             } elsif (is_valid_drivename($opt)) {
4819                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4820                 # some changes can be done without hotplug
4821                 my $drive = parse_drive($opt, $value);
4822                 if (drive_is_cloudinit($drive)) {
4823                     &$apply_pending_cloudinit($opt, $value);
4824                 }
4825                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4826                                      $vmid, $opt, $value, $arch, $machine_type);
4827             } elsif ($opt =~ m/^memory$/) { #dimms
4828                 die "skip\n" if !$hotplug_features->{memory};
4829                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4830             } elsif ($opt eq 'cpuunits') {
4831                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4832             } elsif ($opt eq 'cpulimit') {
4833                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4834                 $cgroup->change_cpu_quota($cpulimit, 100000);
4835             } else {
4836                 die "skip\n";  # skip non-hot-pluggable options
4837             }
4838         };
4839         if (my $err = $@) {
4840             &$add_error($opt, $err) if $err ne "skip\n";
4841         } else {
4842             $conf->{$opt} = $value;
4843             delete $conf->{pending}->{$opt};
4844         }
4845     }
4846
4847     PVE::QemuConfig->write_config($vmid, $conf);
4848 }
4849
4850 sub try_deallocate_drive {
4851     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4852
4853     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4854         my $volid = $drive->{file};
4855         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4856             my $sid = PVE::Storage::parse_volume_id($volid);
4857             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4858
4859             # check if the disk is really unused
4860             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4861                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4862             PVE::Storage::vdisk_free($storecfg, $volid);
4863             return 1;
4864         } else {
4865             # If vm is not owner of this disk remove from config
4866             return 1;
4867         }
4868     }
4869
4870     return;
4871 }
4872
4873 sub vmconfig_delete_or_detach_drive {
4874     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4875
4876     my $drive = parse_drive($opt, $conf->{$opt});
4877
4878     my $rpcenv = PVE::RPCEnvironment::get();
4879     my $authuser = $rpcenv->get_user();
4880
4881     if ($force) {
4882         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4883         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4884     } else {
4885         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4886     }
4887 }
4888
4889
4890
4891 sub vmconfig_apply_pending {
4892     my ($vmid, $conf, $storecfg, $errors) = @_;
4893
4894     my $add_apply_error = sub {
4895         my ($opt, $msg) = @_;
4896         my $err_msg = "unable to apply pending change $opt : $msg";
4897         $errors->{$opt} = $err_msg;
4898         warn $err_msg;
4899     };
4900
4901     # cold plug
4902
4903     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4904     foreach my $opt (sort keys %$pending_delete_hash) {
4905         my $force = $pending_delete_hash->{$opt}->{force};
4906         eval {
4907             if ($opt =~ m/^unused/) {
4908                 die "internal error";
4909             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4910                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4911             }
4912         };
4913         if (my $err = $@) {
4914             $add_apply_error->($opt, $err);
4915         } else {
4916             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4917             delete $conf->{$opt};
4918         }
4919     }
4920
4921     PVE::QemuConfig->cleanup_pending($conf);
4922
4923     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4924         next if $opt eq 'delete'; # just to be sure
4925         eval {
4926             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4927                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4928             }
4929         };
4930         if (my $err = $@) {
4931             $add_apply_error->($opt, $err);
4932         } else {
4933             $conf->{$opt} = delete $conf->{pending}->{$opt};
4934         }
4935     }
4936
4937     # write all changes at once to avoid unnecessary i/o
4938     PVE::QemuConfig->write_config($vmid, $conf);
4939 }
4940
4941 sub vmconfig_update_net {
4942     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4943
4944     my $newnet = parse_net($value);
4945
4946     if ($conf->{$opt}) {
4947         my $oldnet = parse_net($conf->{$opt});
4948
4949         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4950             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4951             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4952             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4953
4954             # for non online change, we try to hot-unplug
4955             die "skip\n" if !$hotplug;
4956             vm_deviceunplug($vmid, $conf, $opt);
4957         } else {
4958
4959             die "internal error" if $opt !~ m/net(\d+)/;
4960             my $iface = "tap${vmid}i$1";
4961
4962             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4963                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4964                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4965                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4966                 PVE::Network::tap_unplug($iface);
4967
4968                 if ($have_sdn) {
4969                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4970                 } else {
4971                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4972                 }
4973             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4974                 # Rate can be applied on its own but any change above needs to
4975                 # include the rate in tap_plug since OVS resets everything.
4976                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4977             }
4978
4979             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4980                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4981             }
4982
4983             return 1;
4984         }
4985     }
4986
4987     if ($hotplug) {
4988         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4989     } else {
4990         die "skip\n";
4991     }
4992 }
4993
4994 sub vmconfig_update_disk {
4995     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4996
4997     my $drive = parse_drive($opt, $value);
4998
4999     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5000         my $media = $drive->{media} || 'disk';
5001         my $oldmedia = $old_drive->{media} || 'disk';
5002         die "unable to change media type\n" if $media ne $oldmedia;
5003
5004         if (!drive_is_cdrom($old_drive)) {
5005
5006             if ($drive->{file} ne $old_drive->{file}) {
5007
5008                 die "skip\n" if !$hotplug;
5009
5010                 # unplug and register as unused
5011                 vm_deviceunplug($vmid, $conf, $opt);
5012                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5013
5014             } else {
5015                 # update existing disk
5016
5017                 # skip non hotpluggable value
5018                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5019                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5020                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5021                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5022                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5023                     die "skip\n";
5024                 }
5025
5026                 # apply throttle
5027                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5028                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5029                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5030                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5031                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5032                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5033                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5034                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5035                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5036                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5037                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5038                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5039                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5040                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5041                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5042                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5043                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5044                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5045
5046                     qemu_block_set_io_throttle(
5047                         $vmid,"drive-$opt",
5048                         ($drive->{mbps} || 0)*1024*1024,
5049                         ($drive->{mbps_rd} || 0)*1024*1024,
5050                         ($drive->{mbps_wr} || 0)*1024*1024,
5051                          $drive->{iops} || 0,
5052                          $drive->{iops_rd} || 0,
5053                          $drive->{iops_wr} || 0,
5054                         ($drive->{mbps_max} || 0)*1024*1024,
5055                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5056                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5057                          $drive->{iops_max} || 0,
5058                          $drive->{iops_rd_max} || 0,
5059                          $drive->{iops_wr_max} || 0,
5060                          $drive->{bps_max_length} || 1,
5061                          $drive->{bps_rd_max_length} || 1,
5062                          $drive->{bps_wr_max_length} || 1,
5063                          $drive->{iops_max_length} || 1,
5064                          $drive->{iops_rd_max_length} || 1,
5065                          $drive->{iops_wr_max_length} || 1,
5066                     );
5067
5068                 }
5069
5070                 return 1;
5071             }
5072
5073         } else { # cdrom
5074
5075             if ($drive->{file} eq 'none') {
5076                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5077                 if (drive_is_cloudinit($old_drive)) {
5078                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5079                 }
5080             } else {
5081                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5082
5083                 # force eject if locked
5084                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5085
5086                 if ($path) {
5087                     mon_cmd($vmid, "blockdev-change-medium",
5088                         id => "$opt", filename => "$path");
5089                 }
5090             }
5091
5092             return 1;
5093         }
5094     }
5095
5096     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5097     # hotplug new disks
5098     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5099     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5100 }
5101
5102 # called in locked context by incoming migration
5103 sub vm_migrate_get_nbd_disks {
5104     my ($storecfg, $conf, $replicated_volumes) = @_;
5105
5106     my $local_volumes = {};
5107     PVE::QemuConfig->foreach_volume($conf, sub {
5108         my ($ds, $drive) = @_;
5109
5110         return if drive_is_cdrom($drive);
5111
5112         my $volid = $drive->{file};
5113
5114         return if !$volid;
5115
5116         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5117
5118         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5119         return if $scfg->{shared};
5120
5121         # replicated disks re-use existing state via bitmap
5122         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5123         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5124     });
5125     return $local_volumes;
5126 }
5127
5128 # called in locked context by incoming migration
5129 sub vm_migrate_alloc_nbd_disks {
5130     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5131
5132     my $format = undef;
5133
5134     my $nbd = {};
5135     foreach my $opt (sort keys %$source_volumes) {
5136         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5137
5138         if ($use_existing) {
5139             $nbd->{$opt}->{drivestr} = print_drive($drive);
5140             $nbd->{$opt}->{volid} = $volid;
5141             $nbd->{$opt}->{replicated} = 1;
5142             next;
5143         }
5144
5145         # If a remote storage is specified and the format of the original
5146         # volume is not available there, fall back to the default format.
5147         # Otherwise use the same format as the original.
5148         if (!$storagemap->{identity}) {
5149             $storeid = map_storage($storagemap, $storeid);
5150             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5151             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5152             my $fileFormat = qemu_img_format($scfg, $volname);
5153             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5154         } else {
5155             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5156             $format = qemu_img_format($scfg, $volname);
5157         }
5158
5159         my $size = $drive->{size} / 1024;
5160         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5161         my $newdrive = $drive;
5162         $newdrive->{format} = $format;
5163         $newdrive->{file} = $newvolid;
5164         my $drivestr = print_drive($newdrive);
5165         $nbd->{$opt}->{drivestr} = $drivestr;
5166         $nbd->{$opt}->{volid} = $newvolid;
5167     }
5168
5169     return $nbd;
5170 }
5171
5172 # see vm_start_nolock for parameters, additionally:
5173 # migrate_opts:
5174 #   storagemap = parsed storage map for allocating NBD disks
5175 sub vm_start {
5176     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5177
5178     return PVE::QemuConfig->lock_config($vmid, sub {
5179         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5180
5181         die "you can't start a vm if it's a template\n"
5182             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5183
5184         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5185         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5186
5187         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5188
5189         if ($has_backup_lock && $running) {
5190             # a backup is currently running, attempt to start the guest in the
5191             # existing QEMU instance
5192             return vm_resume($vmid);
5193         }
5194
5195         PVE::QemuConfig->check_lock($conf)
5196             if !($params->{skiplock} || $has_suspended_lock);
5197
5198         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5199
5200         die "VM $vmid already running\n" if $running;
5201
5202         if (my $storagemap = $migrate_opts->{storagemap}) {
5203             my $replicated = $migrate_opts->{replicated_volumes};
5204             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5205             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5206
5207             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5208                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5209             }
5210         }
5211
5212         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5213     });
5214 }
5215
5216
5217 # params:
5218 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5219 #   skiplock => 0/1, skip checking for config lock
5220 #   skiptemplate => 0/1, skip checking whether VM is template
5221 #   forcemachine => to force Qemu machine (rollback/migration)
5222 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5223 #   timeout => in seconds
5224 #   paused => start VM in paused state (backup)
5225 #   resume => resume from hibernation
5226 #   pbs-backing => {
5227 #      sata0 => {
5228 #         repository
5229 #         snapshot
5230 #         keyfile
5231 #         archive
5232 #      },
5233 #      virtio2 => ...
5234 #   }
5235 # migrate_opts:
5236 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5237 #   migratedfrom => source node
5238 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5239 #   network => CIDR of migration network
5240 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5241 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5242 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5243 sub vm_start_nolock {
5244     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5245
5246     my $statefile = $params->{statefile};
5247     my $resume = $params->{resume};
5248
5249     my $migratedfrom = $migrate_opts->{migratedfrom};
5250     my $migration_type = $migrate_opts->{type};
5251
5252     my $res = {};
5253
5254     # clean up leftover reboot request files
5255     eval { clear_reboot_request($vmid); };
5256     warn $@ if $@;
5257
5258     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5259         vmconfig_apply_pending($vmid, $conf, $storecfg);
5260         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5261     }
5262
5263     # don't regenerate the ISO if the VM is started as part of a live migration
5264     # this way we can reuse the old ISO with the correct config
5265     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5266
5267     my $defaults = load_defaults();
5268
5269     # set environment variable useful inside network script
5270     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5271
5272     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5273
5274     my $forcemachine = $params->{forcemachine};
5275     my $forcecpu = $params->{forcecpu};
5276     if ($resume) {
5277         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5278         $forcemachine = $conf->{runningmachine};
5279         $forcecpu = $conf->{runningcpu};
5280         print "Resuming suspended VM\n";
5281     }
5282
5283     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5284         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5285
5286     my $migration_ip;
5287     my $get_migration_ip = sub {
5288         my ($nodename) = @_;
5289
5290         return $migration_ip if defined($migration_ip);
5291
5292         my $cidr = $migrate_opts->{network};
5293
5294         if (!defined($cidr)) {
5295             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5296             $cidr = $dc_conf->{migration}->{network};
5297         }
5298
5299         if (defined($cidr)) {
5300             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5301
5302             die "could not get IP: no address configured on local " .
5303                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5304
5305             die "could not get IP: multiple addresses configured on local " .
5306                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5307
5308             $migration_ip = @$ips[0];
5309         }
5310
5311         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5312             if !defined($migration_ip);
5313
5314         return $migration_ip;
5315     };
5316
5317     my $migrate_uri;
5318     if ($statefile) {
5319         if ($statefile eq 'tcp') {
5320             my $localip = "localhost";
5321             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5322             my $nodename = nodename();
5323
5324             if (!defined($migration_type)) {
5325                 if (defined($datacenterconf->{migration}->{type})) {
5326                     $migration_type = $datacenterconf->{migration}->{type};
5327                 } else {
5328                     $migration_type = 'secure';
5329                 }
5330             }
5331
5332             if ($migration_type eq 'insecure') {
5333                 $localip = $get_migration_ip->($nodename);
5334                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5335             }
5336
5337             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5338             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5339             $migrate_uri = "tcp:${localip}:${migrate_port}";
5340             push @$cmd, '-incoming', $migrate_uri;
5341             push @$cmd, '-S';
5342
5343         } elsif ($statefile eq 'unix') {
5344             # should be default for secure migrations as a ssh TCP forward
5345             # tunnel is not deterministic reliable ready and fails regurarly
5346             # to set up in time, so use UNIX socket forwards
5347             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5348             unlink $socket_addr;
5349
5350             $migrate_uri = "unix:$socket_addr";
5351
5352             push @$cmd, '-incoming', $migrate_uri;
5353             push @$cmd, '-S';
5354
5355         } elsif (-e $statefile) {
5356             push @$cmd, '-loadstate', $statefile;
5357         } else {
5358             my $statepath = PVE::Storage::path($storecfg, $statefile);
5359             push @$vollist, $statefile;
5360             push @$cmd, '-loadstate', $statepath;
5361         }
5362     } elsif ($params->{paused}) {
5363         push @$cmd, '-S';
5364     }
5365
5366     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5367
5368     my $pci_devices = {}; # host pci devices
5369     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5370         my $dev = $conf->{"hostpci$i"} or next;
5371         $pci_devices->{$i} = parse_hostpci($dev);
5372     }
5373
5374     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } values $pci_devices->%* ];
5375     # reserve all PCI IDs before actually doing anything with them
5376     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5377
5378     eval {
5379         for my $id (sort keys %$pci_devices) {
5380             my $d = $pci_devices->{$id};
5381             for my $dev ($d->{pciid}->@*) {
5382                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5383             }
5384         }
5385     };
5386     if (my $err = $@) {
5387         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5388         warn $@ if $@;
5389         die $err;
5390     }
5391
5392     PVE::Storage::activate_volumes($storecfg, $vollist);
5393
5394     eval {
5395         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5396     };
5397     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5398     # timeout should be more than enough here...
5399     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5400
5401     my $cpuunits = get_cpuunits($conf);
5402
5403     my %run_params = (
5404         timeout => $statefile ? undef : $start_timeout,
5405         umask => 0077,
5406         noerr => 1,
5407     );
5408
5409     # when migrating, prefix QEMU output so other side can pick up any
5410     # errors that might occur and show the user
5411     if ($migratedfrom) {
5412         $run_params{quiet} = 1;
5413         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5414     }
5415
5416     my %systemd_properties = (
5417         Slice => 'qemu.slice',
5418         KillMode => 'process',
5419         SendSIGKILL => 0,
5420         TimeoutStopUSec => ULONG_MAX, # infinity
5421     );
5422
5423     if (PVE::CGroup::cgroup_mode() == 2) {
5424         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5425         $systemd_properties{CPUWeight} = $cpuunits;
5426     } else {
5427         $systemd_properties{CPUShares} = $cpuunits;
5428     }
5429
5430     if (my $cpulimit = $conf->{cpulimit}) {
5431         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5432     }
5433     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5434
5435     my $run_qemu = sub {
5436         PVE::Tools::run_fork sub {
5437             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5438
5439             my $tpmpid;
5440             if (my $tpm = $conf->{tpmstate0}) {
5441                 # start the TPM emulator so QEMU can connect on start
5442                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5443             }
5444
5445             my $exitcode = run_command($cmd, %run_params);
5446             if ($exitcode) {
5447                 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5448                 kill 'TERM', $tpmpid if $tpmpid;
5449                 die "QEMU exited with code $exitcode\n";
5450             }
5451         };
5452     };
5453
5454     if ($conf->{hugepages}) {
5455
5456         my $code = sub {
5457             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5458             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5459
5460             PVE::QemuServer::Memory::hugepages_mount();
5461             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5462
5463             eval { $run_qemu->() };
5464             if (my $err = $@) {
5465                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5466                     if !$conf->{keephugepages};
5467                 die $err;
5468             }
5469
5470             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5471                 if !$conf->{keephugepages};
5472         };
5473         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5474
5475     } else {
5476         eval { $run_qemu->() };
5477     }
5478
5479     if (my $err = $@) {
5480         # deactivate volumes if start fails
5481         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5482         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5483
5484         die "start failed: $err";
5485     }
5486
5487     # re-reserve all PCI IDs now that we can know the actual VM PID
5488     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5489     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5490     warn $@ if $@;
5491
5492     print "migration listens on $migrate_uri\n" if $migrate_uri;
5493     $res->{migrate_uri} = $migrate_uri;
5494
5495     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5496         eval { mon_cmd($vmid, "cont"); };
5497         warn $@ if $@;
5498     }
5499
5500     #start nbd server for storage migration
5501     if (my $nbd = $migrate_opts->{nbd}) {
5502         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5503
5504         my $migrate_storage_uri;
5505         # nbd_protocol_version > 0 for unix socket support
5506         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5507             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5508             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5509             $migrate_storage_uri = "nbd:unix:$socket_path";
5510         } else {
5511             my $nodename = nodename();
5512             my $localip = $get_migration_ip->($nodename);
5513             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5514             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5515
5516             mon_cmd($vmid, "nbd-server-start", addr => {
5517                 type => 'inet',
5518                 data => {
5519                     host => "${localip}",
5520                     port => "${storage_migrate_port}",
5521                 },
5522             });
5523             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5524             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5525         }
5526
5527         $res->{migrate_storage_uri} = $migrate_storage_uri;
5528
5529         foreach my $opt (sort keys %$nbd) {
5530             my $drivestr = $nbd->{$opt}->{drivestr};
5531             my $volid = $nbd->{$opt}->{volid};
5532             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5533             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5534             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5535             print "re-using replicated volume: $opt - $volid\n"
5536                 if $nbd->{$opt}->{replicated};
5537
5538             $res->{drives}->{$opt} = $nbd->{$opt};
5539             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5540         }
5541     }
5542
5543     if ($migratedfrom) {
5544         eval {
5545             set_migration_caps($vmid);
5546         };
5547         warn $@ if $@;
5548
5549         if ($spice_port) {
5550             print "spice listens on port $spice_port\n";
5551             $res->{spice_port} = $spice_port;
5552             if ($migrate_opts->{spice_ticket}) {
5553                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5554                     $migrate_opts->{spice_ticket});
5555                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5556             }
5557         }
5558
5559     } else {
5560         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5561             if !$statefile && $conf->{balloon};
5562
5563         foreach my $opt (keys %$conf) {
5564             next if $opt !~  m/^net\d+$/;
5565             my $nicconf = parse_net($conf->{$opt});
5566             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5567         }
5568     }
5569
5570     mon_cmd($vmid, 'qom-set',
5571                 path => "machine/peripheral/balloon0",
5572                 property => "guest-stats-polling-interval",
5573                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5574
5575     if ($resume) {
5576         print "Resumed VM, removing state\n";
5577         if (my $vmstate = $conf->{vmstate}) {
5578             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5579             PVE::Storage::vdisk_free($storecfg, $vmstate);
5580         }
5581         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5582         PVE::QemuConfig->write_config($vmid, $conf);
5583     }
5584
5585     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5586
5587     return $res;
5588 }
5589
5590 sub vm_commandline {
5591     my ($storecfg, $vmid, $snapname) = @_;
5592
5593     my $conf = PVE::QemuConfig->load_config($vmid);
5594     my $forcemachine;
5595     my $forcecpu;
5596
5597     if ($snapname) {
5598         my $snapshot = $conf->{snapshots}->{$snapname};
5599         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5600
5601         # check for machine or CPU overrides in snapshot
5602         $forcemachine = $snapshot->{runningmachine};
5603         $forcecpu = $snapshot->{runningcpu};
5604
5605         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5606
5607         $conf = $snapshot;
5608     }
5609
5610     my $defaults = load_defaults();
5611
5612     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5613         $forcemachine, $forcecpu);
5614
5615     return PVE::Tools::cmd2string($cmd);
5616 }
5617
5618 sub vm_reset {
5619     my ($vmid, $skiplock) = @_;
5620
5621     PVE::QemuConfig->lock_config($vmid, sub {
5622
5623         my $conf = PVE::QemuConfig->load_config($vmid);
5624
5625         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5626
5627         mon_cmd($vmid, "system_reset");
5628     });
5629 }
5630
5631 sub get_vm_volumes {
5632     my ($conf) = @_;
5633
5634     my $vollist = [];
5635     foreach_volid($conf, sub {
5636         my ($volid, $attr) = @_;
5637
5638         return if $volid =~ m|^/|;
5639
5640         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5641         return if !$sid;
5642
5643         push @$vollist, $volid;
5644     });
5645
5646     return $vollist;
5647 }
5648
5649 sub vm_stop_cleanup {
5650     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5651
5652     eval {
5653
5654         if (!$keepActive) {
5655             my $vollist = get_vm_volumes($conf);
5656             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5657
5658             if (my $tpmdrive = $conf->{tpmstate0}) {
5659                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5660                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5661                 if ($storeid) {
5662                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5663                 }
5664             }
5665         }
5666
5667         foreach my $ext (qw(mon qmp pid vnc qga)) {
5668             unlink "/var/run/qemu-server/${vmid}.$ext";
5669         }
5670
5671         if ($conf->{ivshmem}) {
5672             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5673             # just delete it for now, VMs which have this already open do not
5674             # are affected, but new VMs will get a separated one. If this
5675             # becomes an issue we either add some sort of ref-counting or just
5676             # add a "don't delete on stop" flag to the ivshmem format.
5677             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5678         }
5679
5680         my $ids = [];
5681         foreach my $key (keys %$conf) {
5682             next if $key !~ m/^hostpci(\d+)$/;
5683             my $hostpciindex = $1;
5684             my $d = parse_hostpci($conf->{$key});
5685             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5686
5687             foreach my $pci (@{$d->{pciid}}) {
5688                 my $pciid = $pci->{id};
5689                 push @$ids, $pci->{id};
5690                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5691             }
5692         }
5693         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5694
5695         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5696     };
5697     warn $@ if $@; # avoid errors - just warn
5698 }
5699
5700 # call only in locked context
5701 sub _do_vm_stop {
5702     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5703
5704     my $pid = check_running($vmid, $nocheck);
5705     return if !$pid;
5706
5707     my $conf;
5708     if (!$nocheck) {
5709         $conf = PVE::QemuConfig->load_config($vmid);
5710         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5711         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5712             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5713             $timeout = $opts->{down} if $opts->{down};
5714         }
5715         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5716     }
5717
5718     eval {
5719         if ($shutdown) {
5720             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5721                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5722             } else {
5723                 mon_cmd($vmid, "system_powerdown");
5724             }
5725         } else {
5726             mon_cmd($vmid, "quit");
5727         }
5728     };
5729     my $err = $@;
5730
5731     if (!$err) {
5732         $timeout = 60 if !defined($timeout);
5733
5734         my $count = 0;
5735         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5736             $count++;
5737             sleep 1;
5738         }
5739
5740         if ($count >= $timeout) {
5741             if ($force) {
5742                 warn "VM still running - terminating now with SIGTERM\n";
5743                 kill 15, $pid;
5744             } else {
5745                 die "VM quit/powerdown failed - got timeout\n";
5746             }
5747         } else {
5748             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5749             return;
5750         }
5751     } else {
5752        if (!check_running($vmid, $nocheck)) {
5753             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5754             return;
5755        }
5756        if ($force) {
5757             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5758             kill 15, $pid;
5759         } else {
5760             die "VM quit/powerdown failed\n";
5761         }
5762     }
5763
5764     # wait again
5765     $timeout = 10;
5766
5767     my $count = 0;
5768     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5769         $count++;
5770         sleep 1;
5771     }
5772
5773     if ($count >= $timeout) {
5774         warn "VM still running - terminating now with SIGKILL\n";
5775         kill 9, $pid;
5776         sleep 1;
5777     }
5778
5779     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5780 }
5781
5782 # Note: use $nocheck to skip tests if VM configuration file exists.
5783 # We need that when migration VMs to other nodes (files already moved)
5784 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5785 sub vm_stop {
5786     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5787
5788     $force = 1 if !defined($force) && !$shutdown;
5789
5790     if ($migratedfrom){
5791         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5792         kill 15, $pid if $pid;
5793         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5794         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5795         return;
5796     }
5797
5798     PVE::QemuConfig->lock_config($vmid, sub {
5799         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5800    });
5801 }
5802
5803 sub vm_reboot {
5804     my ($vmid, $timeout) = @_;
5805
5806     PVE::QemuConfig->lock_config($vmid, sub {
5807         eval {
5808
5809             # only reboot if running, as qmeventd starts it again on a stop event
5810             return if !check_running($vmid);
5811
5812             create_reboot_request($vmid);
5813
5814             my $storecfg = PVE::Storage::config();
5815             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5816
5817         };
5818         if (my $err = $@) {
5819             # avoid that the next normal shutdown will be confused for a reboot
5820             clear_reboot_request($vmid);
5821             die $err;
5822         }
5823    });
5824 }
5825
5826 # note: if using the statestorage parameter, the caller has to check privileges
5827 sub vm_suspend {
5828     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5829
5830     my $conf;
5831     my $path;
5832     my $storecfg;
5833     my $vmstate;
5834
5835     PVE::QemuConfig->lock_config($vmid, sub {
5836
5837         $conf = PVE::QemuConfig->load_config($vmid);
5838
5839         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5840         PVE::QemuConfig->check_lock($conf)
5841             if !($skiplock || $is_backing_up);
5842
5843         die "cannot suspend to disk during backup\n"
5844             if $is_backing_up && $includestate;
5845
5846         if ($includestate) {
5847             $conf->{lock} = 'suspending';
5848             my $date = strftime("%Y-%m-%d", localtime(time()));
5849             $storecfg = PVE::Storage::config();
5850             if (!$statestorage) {
5851                 $statestorage = find_vmstate_storage($conf, $storecfg);
5852                 # check permissions for the storage
5853                 my $rpcenv = PVE::RPCEnvironment::get();
5854                 if ($rpcenv->{type} ne 'cli') {
5855                     my $authuser = $rpcenv->get_user();
5856                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5857                 }
5858             }
5859
5860
5861             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5862                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5863             $path = PVE::Storage::path($storecfg, $vmstate);
5864             PVE::QemuConfig->write_config($vmid, $conf);
5865         } else {
5866             mon_cmd($vmid, "stop");
5867         }
5868     });
5869
5870     if ($includestate) {
5871         # save vm state
5872         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5873
5874         eval {
5875             set_migration_caps($vmid, 1);
5876             mon_cmd($vmid, "savevm-start", statefile => $path);
5877             for(;;) {
5878                 my $state = mon_cmd($vmid, "query-savevm");
5879                 if (!$state->{status}) {
5880                     die "savevm not active\n";
5881                 } elsif ($state->{status} eq 'active') {
5882                     sleep(1);
5883                     next;
5884                 } elsif ($state->{status} eq 'completed') {
5885                     print "State saved, quitting\n";
5886                     last;
5887                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5888                     die "query-savevm failed with error '$state->{error}'\n"
5889                 } else {
5890                     die "query-savevm returned status '$state->{status}'\n";
5891                 }
5892             }
5893         };
5894         my $err = $@;
5895
5896         PVE::QemuConfig->lock_config($vmid, sub {
5897             $conf = PVE::QemuConfig->load_config($vmid);
5898             if ($err) {
5899                 # cleanup, but leave suspending lock, to indicate something went wrong
5900                 eval {
5901                     mon_cmd($vmid, "savevm-end");
5902                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5903                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5904                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5905                     PVE::QemuConfig->write_config($vmid, $conf);
5906                 };
5907                 warn $@ if $@;
5908                 die $err;
5909             }
5910
5911             die "lock changed unexpectedly\n"
5912                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5913
5914             mon_cmd($vmid, "quit");
5915             $conf->{lock} = 'suspended';
5916             PVE::QemuConfig->write_config($vmid, $conf);
5917         });
5918     }
5919 }
5920
5921 sub vm_resume {
5922     my ($vmid, $skiplock, $nocheck) = @_;
5923
5924     PVE::QemuConfig->lock_config($vmid, sub {
5925         my $res = mon_cmd($vmid, 'query-status');
5926         my $resume_cmd = 'cont';
5927         my $reset = 0;
5928
5929         if ($res->{status}) {
5930             return if $res->{status} eq 'running'; # job done, go home
5931             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5932             $reset = 1 if $res->{status} eq 'shutdown';
5933         }
5934
5935         if (!$nocheck) {
5936
5937             my $conf = PVE::QemuConfig->load_config($vmid);
5938
5939             PVE::QemuConfig->check_lock($conf)
5940                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5941         }
5942
5943         if ($reset) {
5944             # required if a VM shuts down during a backup and we get a resume
5945             # request before the backup finishes for example
5946             mon_cmd($vmid, "system_reset");
5947         }
5948         mon_cmd($vmid, $resume_cmd);
5949     });
5950 }
5951
5952 sub vm_sendkey {
5953     my ($vmid, $skiplock, $key) = @_;
5954
5955     PVE::QemuConfig->lock_config($vmid, sub {
5956
5957         my $conf = PVE::QemuConfig->load_config($vmid);
5958
5959         # there is no qmp command, so we use the human monitor command
5960         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5961         die $res if $res ne '';
5962     });
5963 }
5964
5965 # vzdump restore implementaion
5966
5967 sub tar_archive_read_firstfile {
5968     my $archive = shift;
5969
5970     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5971
5972     # try to detect archive type first
5973     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5974         die "unable to open file '$archive'\n";
5975     my $firstfile = <$fh>;
5976     kill 15, $pid;
5977     close $fh;
5978
5979     die "ERROR: archive contaions no data\n" if !$firstfile;
5980     chomp $firstfile;
5981
5982     return $firstfile;
5983 }
5984
5985 sub tar_restore_cleanup {
5986     my ($storecfg, $statfile) = @_;
5987
5988     print STDERR "starting cleanup\n";
5989
5990     if (my $fd = IO::File->new($statfile, "r")) {
5991         while (defined(my $line = <$fd>)) {
5992             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5993                 my $volid = $2;
5994                 eval {
5995                     if ($volid =~ m|^/|) {
5996                         unlink $volid || die 'unlink failed\n';
5997                     } else {
5998                         PVE::Storage::vdisk_free($storecfg, $volid);
5999                     }
6000                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6001                 };
6002                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6003             } else {
6004                 print STDERR "unable to parse line in statfile - $line";
6005             }
6006         }
6007         $fd->close();
6008     }
6009 }
6010
6011 sub restore_file_archive {
6012     my ($archive, $vmid, $user, $opts) = @_;
6013
6014     return restore_vma_archive($archive, $vmid, $user, $opts)
6015         if $archive eq '-';
6016
6017     my $info = PVE::Storage::archive_info($archive);
6018     my $format = $opts->{format} // $info->{format};
6019     my $comp = $info->{compression};
6020
6021     # try to detect archive format
6022     if ($format eq 'tar') {
6023         return restore_tar_archive($archive, $vmid, $user, $opts);
6024     } else {
6025         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6026     }
6027 }
6028
6029 # hepler to remove disks that will not be used after restore
6030 my $restore_cleanup_oldconf = sub {
6031     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6032
6033     PVE::QemuConfig->foreach_volume($oldconf, sub {
6034         my ($ds, $drive) = @_;
6035
6036         return if drive_is_cdrom($drive, 1);
6037
6038         my $volid = $drive->{file};
6039         return if !$volid || $volid =~ m|^/|;
6040
6041         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6042         return if !$path || !$owner || ($owner != $vmid);
6043
6044         # Note: only delete disk we want to restore
6045         # other volumes will become unused
6046         if ($virtdev_hash->{$ds}) {
6047             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6048             if (my $err = $@) {
6049                 warn $err;
6050             }
6051         }
6052     });
6053
6054     # delete vmstate files, after the restore we have no snapshots anymore
6055     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6056         my $snap = $oldconf->{snapshots}->{$snapname};
6057         if ($snap->{vmstate}) {
6058             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6059             if (my $err = $@) {
6060                 warn $err;
6061             }
6062         }
6063     }
6064 };
6065
6066 # Helper to parse vzdump backup device hints
6067 #
6068 # $rpcenv: Environment, used to ckeck storage permissions
6069 # $user: User ID, to check storage permissions
6070 # $storecfg: Storage configuration
6071 # $fh: the file handle for reading the configuration
6072 # $devinfo: should contain device sizes for all backu-up'ed devices
6073 # $options: backup options (pool, default storage)
6074 #
6075 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6076 my $parse_backup_hints = sub {
6077     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6078
6079     my $virtdev_hash = {};
6080
6081     while (defined(my $line = <$fh>)) {
6082         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6083             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6084             die "archive does not contain data for drive '$virtdev'\n"
6085                 if !$devinfo->{$devname};
6086
6087             if (defined($options->{storage})) {
6088                 $storeid = $options->{storage} || 'local';
6089             } elsif (!$storeid) {
6090                 $storeid = 'local';
6091             }
6092             $format = 'raw' if !$format;
6093             $devinfo->{$devname}->{devname} = $devname;
6094             $devinfo->{$devname}->{virtdev} = $virtdev;
6095             $devinfo->{$devname}->{format} = $format;
6096             $devinfo->{$devname}->{storeid} = $storeid;
6097
6098             # check permission on storage
6099             my $pool = $options->{pool}; # todo: do we need that?
6100             if ($user ne 'root@pam') {
6101                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6102             }
6103
6104             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6105         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6106             my $virtdev = $1;
6107             my $drive = parse_drive($virtdev, $2);
6108             if (drive_is_cloudinit($drive)) {
6109                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6110                 $storeid = $options->{storage} if defined ($options->{storage});
6111                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6112                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6113
6114                 $virtdev_hash->{$virtdev} = {
6115                     format => $format,
6116                     storeid => $storeid,
6117                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6118                     is_cloudinit => 1,
6119                 };
6120             }
6121         }
6122     }
6123
6124     return $virtdev_hash;
6125 };
6126
6127 # Helper to allocate and activate all volumes required for a restore
6128 #
6129 # $storecfg: Storage configuration
6130 # $virtdev_hash: as returned by parse_backup_hints()
6131 #
6132 # Returns: { $virtdev => $volid }
6133 my $restore_allocate_devices = sub {
6134     my ($storecfg, $virtdev_hash, $vmid) = @_;
6135
6136     my $map = {};
6137     foreach my $virtdev (sort keys %$virtdev_hash) {
6138         my $d = $virtdev_hash->{$virtdev};
6139         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6140         my $storeid = $d->{storeid};
6141         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6142
6143         # test if requested format is supported
6144         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6145         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6146         $d->{format} = $defFormat if !$supported;
6147
6148         my $name;
6149         if ($d->{is_cloudinit}) {
6150             $name = "vm-$vmid-cloudinit";
6151             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6152             if ($scfg->{path}) {
6153                 $name .= ".$d->{format}";
6154             }
6155         }
6156
6157         my $volid = PVE::Storage::vdisk_alloc(
6158             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6159
6160         print STDERR "new volume ID is '$volid'\n";
6161         $d->{volid} = $volid;
6162
6163         PVE::Storage::activate_volumes($storecfg, [$volid]);
6164
6165         $map->{$virtdev} = $volid;
6166     }
6167
6168     return $map;
6169 };
6170
6171 sub restore_update_config_line {
6172     my ($cookie, $map, $line, $unique) = @_;
6173
6174     return '' if $line =~ m/^\#qmdump\#/;
6175     return '' if $line =~ m/^\#vzdump\#/;
6176     return '' if $line =~ m/^lock:/;
6177     return '' if $line =~ m/^unused\d+:/;
6178     return '' if $line =~ m/^parent:/;
6179
6180     my $res = '';
6181
6182     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6183     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6184         # try to convert old 1.X settings
6185         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6186         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6187             my ($model, $macaddr) = split(/\=/, $devconfig);
6188             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6189             my $net = {
6190                 model => $model,
6191                 bridge => "vmbr$ind",
6192                 macaddr => $macaddr,
6193             };
6194             my $netstr = print_net($net);
6195
6196             $res .= "net$cookie->{netcount}: $netstr\n";
6197             $cookie->{netcount}++;
6198         }
6199     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6200         my ($id, $netstr) = ($1, $2);
6201         my $net = parse_net($netstr);
6202         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6203         $netstr = print_net($net);
6204         $res .= "$id: $netstr\n";
6205     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6206         my $virtdev = $1;
6207         my $value = $3;
6208         my $di = parse_drive($virtdev, $value);
6209         if (defined($di->{backup}) && !$di->{backup}) {
6210             $res .= "#$line";
6211         } elsif ($map->{$virtdev}) {
6212             delete $di->{format}; # format can change on restore
6213             $di->{file} = $map->{$virtdev};
6214             $value = print_drive($di);
6215             $res .= "$virtdev: $value\n";
6216         } else {
6217             $res .= $line;
6218         }
6219     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6220         my $vmgenid = $1;
6221         if ($vmgenid ne '0') {
6222             # always generate a new vmgenid if there was a valid one setup
6223             $vmgenid = generate_uuid();
6224         }
6225         $res .= "vmgenid: $vmgenid\n";
6226     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6227         my ($uuid, $uuid_str);
6228         UUID::generate($uuid);
6229         UUID::unparse($uuid, $uuid_str);
6230         my $smbios1 = parse_smbios1($2);
6231         $smbios1->{uuid} = $uuid_str;
6232         $res .= $1.print_smbios1($smbios1)."\n";
6233     } else {
6234         $res .= $line;
6235     }
6236
6237     return $res;
6238 }
6239
6240 my $restore_deactivate_volumes = sub {
6241     my ($storecfg, $devinfo) = @_;
6242
6243     my $vollist = [];
6244     foreach my $devname (keys %$devinfo) {
6245         my $volid = $devinfo->{$devname}->{volid};
6246         push @$vollist, $volid if $volid;
6247     }
6248
6249     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6250 };
6251
6252 my $restore_destroy_volumes = sub {
6253     my ($storecfg, $devinfo) = @_;
6254
6255     foreach my $devname (keys %$devinfo) {
6256         my $volid = $devinfo->{$devname}->{volid};
6257         next if !$volid;
6258         eval {
6259             if ($volid =~ m|^/|) {
6260                 unlink $volid || die 'unlink failed\n';
6261             } else {
6262                 PVE::Storage::vdisk_free($storecfg, $volid);
6263             }
6264             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6265         };
6266         print STDERR "unable to cleanup '$volid' - $@" if $@;
6267     }
6268 };
6269
6270 sub scan_volids {
6271     my ($cfg, $vmid) = @_;
6272
6273     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6274
6275     my $volid_hash = {};
6276     foreach my $storeid (keys %$info) {
6277         foreach my $item (@{$info->{$storeid}}) {
6278             next if !($item->{volid} && $item->{size});
6279             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6280             $volid_hash->{$item->{volid}} = $item;
6281         }
6282     }
6283
6284     return $volid_hash;
6285 }
6286
6287 sub update_disk_config {
6288     my ($vmid, $conf, $volid_hash) = @_;
6289
6290     my $changes;
6291     my $prefix = "VM $vmid";
6292
6293     # used and unused disks
6294     my $referenced = {};
6295
6296     # Note: it is allowed to define multiple storages with same path (alias), so
6297     # we need to check both 'volid' and real 'path' (two different volid can point
6298     # to the same path).
6299
6300     my $referencedpath = {};
6301
6302     # update size info
6303     PVE::QemuConfig->foreach_volume($conf, sub {
6304         my ($opt, $drive) = @_;
6305
6306         my $volid = $drive->{file};
6307         return if !$volid;
6308         my $volume = $volid_hash->{$volid};
6309
6310         # mark volid as "in-use" for next step
6311         $referenced->{$volid} = 1;
6312         if ($volume && (my $path = $volume->{path})) {
6313             $referencedpath->{$path} = 1;
6314         }
6315
6316         return if drive_is_cdrom($drive);
6317         return if !$volume;
6318
6319         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6320         if (defined($updated)) {
6321             $changes = 1;
6322             $conf->{$opt} = print_drive($updated);
6323             print "$prefix ($opt): $msg\n";
6324         }
6325     });
6326
6327     # remove 'unusedX' entry if volume is used
6328     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6329         my ($opt, $drive) = @_;
6330
6331         my $volid = $drive->{file};
6332         return if !$volid;
6333
6334         my $path;
6335         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6336         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6337             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6338             $changes = 1;
6339             delete $conf->{$opt};
6340         }
6341
6342         $referenced->{$volid} = 1;
6343         $referencedpath->{$path} = 1 if $path;
6344     });
6345
6346     foreach my $volid (sort keys %$volid_hash) {
6347         next if $volid =~ m/vm-$vmid-state-/;
6348         next if $referenced->{$volid};
6349         my $path = $volid_hash->{$volid}->{path};
6350         next if !$path; # just to be sure
6351         next if $referencedpath->{$path};
6352         $changes = 1;
6353         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6354         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6355         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6356     }
6357
6358     return $changes;
6359 }
6360
6361 sub rescan {
6362     my ($vmid, $nolock, $dryrun) = @_;
6363
6364     my $cfg = PVE::Storage::config();
6365
6366     print "rescan volumes...\n";
6367     my $volid_hash = scan_volids($cfg, $vmid);
6368
6369     my $updatefn =  sub {
6370         my ($vmid) = @_;
6371
6372         my $conf = PVE::QemuConfig->load_config($vmid);
6373
6374         PVE::QemuConfig->check_lock($conf);
6375
6376         my $vm_volids = {};
6377         foreach my $volid (keys %$volid_hash) {
6378             my $info = $volid_hash->{$volid};
6379             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6380         }
6381
6382         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6383
6384         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6385     };
6386
6387     if (defined($vmid)) {
6388         if ($nolock) {
6389             &$updatefn($vmid);
6390         } else {
6391             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6392         }
6393     } else {
6394         my $vmlist = config_list();
6395         foreach my $vmid (keys %$vmlist) {
6396             if ($nolock) {
6397                 &$updatefn($vmid);
6398             } else {
6399                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6400             }
6401         }
6402     }
6403 }
6404
6405 sub restore_proxmox_backup_archive {
6406     my ($archive, $vmid, $user, $options) = @_;
6407
6408     my $storecfg = PVE::Storage::config();
6409
6410     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6411     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6412
6413     my $fingerprint = $scfg->{fingerprint};
6414     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6415
6416     my $repo = PVE::PBSClient::get_repository($scfg);
6417
6418     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6419     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6420     local $ENV{PBS_PASSWORD} = $password;
6421     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6422
6423     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6424         PVE::Storage::parse_volname($storecfg, $archive);
6425
6426     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6427
6428     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6429
6430     my $tmpdir = "/var/tmp/vzdumptmp$$";
6431     rmtree $tmpdir;
6432     mkpath $tmpdir;
6433
6434     my $conffile = PVE::QemuConfig->config_file($vmid);
6435      # disable interrupts (always do cleanups)
6436     local $SIG{INT} =
6437         local $SIG{TERM} =
6438         local $SIG{QUIT} =
6439         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6440
6441     # Note: $oldconf is undef if VM does not exists
6442     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6443     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6444     my $new_conf_raw = '';
6445
6446     my $rpcenv = PVE::RPCEnvironment::get();
6447     my $devinfo = {};
6448
6449     eval {
6450         # enable interrupts
6451         local $SIG{INT} =
6452             local $SIG{TERM} =
6453             local $SIG{QUIT} =
6454             local $SIG{HUP} =
6455             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6456
6457         my $cfgfn = "$tmpdir/qemu-server.conf";
6458         my $firewall_config_fn = "$tmpdir/fw.conf";
6459         my $index_fn = "$tmpdir/index.json";
6460
6461         my $cmd = "restore";
6462
6463         my $param = [$pbs_backup_name, "index.json", $index_fn];
6464         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6465         my $index = PVE::Tools::file_get_contents($index_fn);
6466         $index = decode_json($index);
6467
6468         # print Dumper($index);
6469         foreach my $info (@{$index->{files}}) {
6470             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6471                 my $devname = $1;
6472                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6473                     $devinfo->{$devname}->{size} = $1;
6474                 } else {
6475                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6476                 }
6477             }
6478         }
6479
6480         my $is_qemu_server_backup = scalar(
6481             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6482         );
6483         if (!$is_qemu_server_backup) {
6484             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6485         }
6486         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6487
6488         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6489         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6490
6491         if ($has_firewall_config) {
6492             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6493             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6494
6495             my $pve_firewall_dir = '/etc/pve/firewall';
6496             mkdir $pve_firewall_dir; # make sure the dir exists
6497             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6498         }
6499
6500         my $fh = IO::File->new($cfgfn, "r") ||
6501             die "unable to read qemu-server.conf - $!\n";
6502
6503         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6504
6505         # fixme: rate limit?
6506
6507         # create empty/temp config
6508         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6509
6510         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6511
6512         # allocate volumes
6513         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6514
6515         foreach my $virtdev (sort keys %$virtdev_hash) {
6516             my $d = $virtdev_hash->{$virtdev};
6517             next if $d->{is_cloudinit}; # no need to restore cloudinit
6518
6519             # this fails if storage is unavailable
6520             my $volid = $d->{volid};
6521             my $path = PVE::Storage::path($storecfg, $volid);
6522
6523             # for live-restore we only want to preload the efidisk and TPM state
6524             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6525
6526             my $pbs_restore_cmd = [
6527                 '/usr/bin/pbs-restore',
6528                 '--repository', $repo,
6529                 $pbs_backup_name,
6530                 "$d->{devname}.img.fidx",
6531                 $path,
6532                 '--verbose',
6533                 ];
6534
6535             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6536             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6537
6538             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6539                 push @$pbs_restore_cmd, '--skip-zero';
6540             }
6541
6542             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6543             print "restore proxmox backup image: $dbg_cmdstring\n";
6544             run_command($pbs_restore_cmd);
6545         }
6546
6547         $fh->seek(0, 0) || die "seek failed - $!\n";
6548
6549         my $cookie = { netcount => 0 };
6550         while (defined(my $line = <$fh>)) {
6551             $new_conf_raw .= restore_update_config_line(
6552                 $cookie,
6553                 $map,
6554                 $line,
6555                 $options->{unique},
6556             );
6557         }
6558
6559         $fh->close();
6560     };
6561     my $err = $@;
6562
6563     if ($err || !$options->{live}) {
6564         $restore_deactivate_volumes->($storecfg, $devinfo);
6565     }
6566
6567     rmtree $tmpdir;
6568
6569     if ($err) {
6570         $restore_destroy_volumes->($storecfg, $devinfo);
6571         die $err;
6572     }
6573
6574     if ($options->{live}) {
6575         # keep lock during live-restore
6576         $new_conf_raw .= "\nlock: create";
6577     }
6578
6579     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6580
6581     PVE::Cluster::cfs_update(); # make sure we read new file
6582
6583     eval { rescan($vmid, 1); };
6584     warn $@ if $@;
6585
6586     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6587
6588     if ($options->{live}) {
6589         # enable interrupts
6590         local $SIG{INT} =
6591             local $SIG{TERM} =
6592             local $SIG{QUIT} =
6593             local $SIG{HUP} =
6594             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6595
6596         my $conf = PVE::QemuConfig->load_config($vmid);
6597         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6598
6599         # these special drives are already restored before start
6600         delete $devinfo->{'drive-efidisk0'};
6601         delete $devinfo->{'drive-tpmstate0-backup'};
6602         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6603
6604         PVE::QemuConfig->remove_lock($vmid, "create");
6605     }
6606 }
6607
6608 sub pbs_live_restore {
6609     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6610
6611     print "starting VM for live-restore\n";
6612     print "repository: '$repo', snapshot: '$snap'\n";
6613
6614     my $pbs_backing = {};
6615     for my $ds (keys %$restored_disks) {
6616         $ds =~ m/^drive-(.*)$/;
6617         my $confname = $1;
6618         $pbs_backing->{$confname} = {
6619             repository => $repo,
6620             snapshot => $snap,
6621             archive => "$ds.img.fidx",
6622         };
6623         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6624
6625         my $drive = parse_drive($confname, $conf->{$confname});
6626         print "restoring '$ds' to '$drive->{file}'\n";
6627     }
6628
6629     my $drives_streamed = 0;
6630     eval {
6631         # make sure HA doesn't interrupt our restore by stopping the VM
6632         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6633             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6634         }
6635
6636         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6637         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6638         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6639
6640         my $qmeventd_fd = register_qmeventd_handle($vmid);
6641
6642         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6643         # this will effectively collapse the backing image chain consisting of
6644         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6645         # removes itself once all backing images vanish with 'auto-remove=on')
6646         my $jobs = {};
6647         for my $ds (sort keys %$restored_disks) {
6648             my $job_id = "restore-$ds";
6649             mon_cmd($vmid, 'block-stream',
6650                 'job-id' => $job_id,
6651                 device => "$ds",
6652             );
6653             $jobs->{$job_id} = {};
6654         }
6655
6656         mon_cmd($vmid, 'cont');
6657         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6658
6659         print "restore-drive jobs finished successfully, removing all tracking block devices"
6660             ." to disconnect from Proxmox Backup Server\n";
6661
6662         for my $ds (sort keys %$restored_disks) {
6663             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6664         }
6665
6666         close($qmeventd_fd);
6667     };
6668
6669     my $err = $@;
6670
6671     if ($err) {
6672         warn "An error occured during live-restore: $err\n";
6673         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6674         die "live-restore failed\n";
6675     }
6676 }
6677
6678 sub restore_vma_archive {
6679     my ($archive, $vmid, $user, $opts, $comp) = @_;
6680
6681     my $readfrom = $archive;
6682
6683     my $cfg = PVE::Storage::config();
6684     my $commands = [];
6685     my $bwlimit = $opts->{bwlimit};
6686
6687     my $dbg_cmdstring = '';
6688     my $add_pipe = sub {
6689         my ($cmd) = @_;
6690         push @$commands, $cmd;
6691         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6692         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6693         $readfrom = '-';
6694     };
6695
6696     my $input = undef;
6697     if ($archive eq '-') {
6698         $input = '<&STDIN';
6699     } else {
6700         # If we use a backup from a PVE defined storage we also consider that
6701         # storage's rate limit:
6702         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6703         if (defined($volid)) {
6704             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6705             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6706             if ($readlimit) {
6707                 print STDERR "applying read rate limit: $readlimit\n";
6708                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6709                 $add_pipe->($cstream);
6710             }
6711         }
6712     }
6713
6714     if ($comp) {
6715         my $info = PVE::Storage::decompressor_info('vma', $comp);
6716         my $cmd = $info->{decompressor};
6717         push @$cmd, $readfrom;
6718         $add_pipe->($cmd);
6719     }
6720
6721     my $tmpdir = "/var/tmp/vzdumptmp$$";
6722     rmtree $tmpdir;
6723
6724     # disable interrupts (always do cleanups)
6725     local $SIG{INT} =
6726         local $SIG{TERM} =
6727         local $SIG{QUIT} =
6728         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6729
6730     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6731     POSIX::mkfifo($mapfifo, 0600);
6732     my $fifofh;
6733     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6734
6735     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6736
6737     my $oldtimeout;
6738     my $timeout = 5;
6739
6740     my $devinfo = {};
6741
6742     my $rpcenv = PVE::RPCEnvironment::get();
6743
6744     my $conffile = PVE::QemuConfig->config_file($vmid);
6745
6746     # Note: $oldconf is undef if VM does not exist
6747     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6748     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6749     my $new_conf_raw = '';
6750
6751     my %storage_limits;
6752
6753     my $print_devmap = sub {
6754         my $cfgfn = "$tmpdir/qemu-server.conf";
6755
6756         # we can read the config - that is already extracted
6757         my $fh = IO::File->new($cfgfn, "r") ||
6758             die "unable to read qemu-server.conf - $!\n";
6759
6760         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6761         if (-f $fwcfgfn) {
6762             my $pve_firewall_dir = '/etc/pve/firewall';
6763             mkdir $pve_firewall_dir; # make sure the dir exists
6764             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6765         }
6766
6767         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6768
6769         foreach my $info (values %{$virtdev_hash}) {
6770             my $storeid = $info->{storeid};
6771             next if defined($storage_limits{$storeid});
6772
6773             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6774             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6775             $storage_limits{$storeid} = $limit * 1024;
6776         }
6777
6778         foreach my $devname (keys %$devinfo) {
6779             die "found no device mapping information for device '$devname'\n"
6780                 if !$devinfo->{$devname}->{virtdev};
6781         }
6782
6783         # create empty/temp config
6784         if ($oldconf) {
6785             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6786             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6787         }
6788
6789         # allocate volumes
6790         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6791
6792         # print restore information to $fifofh
6793         foreach my $virtdev (sort keys %$virtdev_hash) {
6794             my $d = $virtdev_hash->{$virtdev};
6795             next if $d->{is_cloudinit}; # no need to restore cloudinit
6796
6797             my $storeid = $d->{storeid};
6798             my $volid = $d->{volid};
6799
6800             my $map_opts = '';
6801             if (my $limit = $storage_limits{$storeid}) {
6802                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6803             }
6804
6805             my $write_zeros = 1;
6806             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6807                 $write_zeros = 0;
6808             }
6809
6810             my $path = PVE::Storage::path($cfg, $volid);
6811
6812             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6813
6814             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6815         }
6816
6817         $fh->seek(0, 0) || die "seek failed - $!\n";
6818
6819         my $cookie = { netcount => 0 };
6820         while (defined(my $line = <$fh>)) {
6821             $new_conf_raw .= restore_update_config_line(
6822                 $cookie,
6823                 $map,
6824                 $line,
6825                 $opts->{unique},
6826             );
6827         }
6828
6829         $fh->close();
6830     };
6831
6832     eval {
6833         # enable interrupts
6834         local $SIG{INT} =
6835             local $SIG{TERM} =
6836             local $SIG{QUIT} =
6837             local $SIG{HUP} =
6838             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6839         local $SIG{ALRM} = sub { die "got timeout\n"; };
6840
6841         $oldtimeout = alarm($timeout);
6842
6843         my $parser = sub {
6844             my $line = shift;
6845
6846             print "$line\n";
6847
6848             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6849                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6850                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6851             } elsif ($line =~ m/^CTIME: /) {
6852                 # we correctly received the vma config, so we can disable
6853                 # the timeout now for disk allocation (set to 10 minutes, so
6854                 # that we always timeout if something goes wrong)
6855                 alarm(600);
6856                 &$print_devmap();
6857                 print $fifofh "done\n";
6858                 my $tmp = $oldtimeout || 0;
6859                 $oldtimeout = undef;
6860                 alarm($tmp);
6861                 close($fifofh);
6862                 $fifofh = undef;
6863             }
6864         };
6865
6866         print "restore vma archive: $dbg_cmdstring\n";
6867         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6868     };
6869     my $err = $@;
6870
6871     alarm($oldtimeout) if $oldtimeout;
6872
6873     $restore_deactivate_volumes->($cfg, $devinfo);
6874
6875     close($fifofh) if $fifofh;
6876     unlink $mapfifo;
6877     rmtree $tmpdir;
6878
6879     if ($err) {
6880         $restore_destroy_volumes->($cfg, $devinfo);
6881         die $err;
6882     }
6883
6884     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6885
6886     PVE::Cluster::cfs_update(); # make sure we read new file
6887
6888     eval { rescan($vmid, 1); };
6889     warn $@ if $@;
6890
6891     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6892 }
6893
6894 sub restore_tar_archive {
6895     my ($archive, $vmid, $user, $opts) = @_;
6896
6897     if ($archive ne '-') {
6898         my $firstfile = tar_archive_read_firstfile($archive);
6899         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6900             if $firstfile ne 'qemu-server.conf';
6901     }
6902
6903     my $storecfg = PVE::Storage::config();
6904
6905     # avoid zombie disks when restoring over an existing VM -> cleanup first
6906     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6907     # skiplock=1 because qmrestore has set the 'create' lock itself already
6908     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6909     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6910
6911     my $tocmd = "/usr/lib/qemu-server/qmextract";
6912
6913     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6914     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6915     $tocmd .= ' --prealloc' if $opts->{prealloc};
6916     $tocmd .= ' --info' if $opts->{info};
6917
6918     # tar option "xf" does not autodetect compression when read from STDIN,
6919     # so we pipe to zcat
6920     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6921         PVE::Tools::shellquote("--to-command=$tocmd");
6922
6923     my $tmpdir = "/var/tmp/vzdumptmp$$";
6924     mkpath $tmpdir;
6925
6926     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6927     local $ENV{VZDUMP_VMID} = $vmid;
6928     local $ENV{VZDUMP_USER} = $user;
6929
6930     my $conffile = PVE::QemuConfig->config_file($vmid);
6931     my $new_conf_raw = '';
6932
6933     # disable interrupts (always do cleanups)
6934     local $SIG{INT} =
6935         local $SIG{TERM} =
6936         local $SIG{QUIT} =
6937         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6938
6939     eval {
6940         # enable interrupts
6941         local $SIG{INT} =
6942             local $SIG{TERM} =
6943             local $SIG{QUIT} =
6944             local $SIG{HUP} =
6945             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6946
6947         if ($archive eq '-') {
6948             print "extracting archive from STDIN\n";
6949             run_command($cmd, input => "<&STDIN");
6950         } else {
6951             print "extracting archive '$archive'\n";
6952             run_command($cmd);
6953         }
6954
6955         return if $opts->{info};
6956
6957         # read new mapping
6958         my $map = {};
6959         my $statfile = "$tmpdir/qmrestore.stat";
6960         if (my $fd = IO::File->new($statfile, "r")) {
6961             while (defined (my $line = <$fd>)) {
6962                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6963                     $map->{$1} = $2 if $1;
6964                 } else {
6965                     print STDERR "unable to parse line in statfile - $line\n";
6966                 }
6967             }
6968             $fd->close();
6969         }
6970
6971         my $confsrc = "$tmpdir/qemu-server.conf";
6972
6973         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6974
6975         my $cookie = { netcount => 0 };
6976         while (defined (my $line = <$srcfd>)) {
6977             $new_conf_raw .= restore_update_config_line(
6978                 $cookie,
6979                 $map,
6980                 $line,
6981                 $opts->{unique},
6982             );
6983         }
6984
6985         $srcfd->close();
6986     };
6987     if (my $err = $@) {
6988         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6989         die $err;
6990     }
6991
6992     rmtree $tmpdir;
6993
6994     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6995
6996     PVE::Cluster::cfs_update(); # make sure we read new file
6997
6998     eval { rescan($vmid, 1); };
6999     warn $@ if $@;
7000 };
7001
7002 sub foreach_storage_used_by_vm {
7003     my ($conf, $func) = @_;
7004
7005     my $sidhash = {};
7006
7007     PVE::QemuConfig->foreach_volume($conf, sub {
7008         my ($ds, $drive) = @_;
7009         return if drive_is_cdrom($drive);
7010
7011         my $volid = $drive->{file};
7012
7013         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7014         $sidhash->{$sid} = $sid if $sid;
7015     });
7016
7017     foreach my $sid (sort keys %$sidhash) {
7018         &$func($sid);
7019     }
7020 }
7021
7022 my $qemu_snap_storage = {
7023     rbd => 1,
7024 };
7025 sub do_snapshots_with_qemu {
7026     my ($storecfg, $volid, $deviceid) = @_;
7027
7028     return if $deviceid =~ m/tpmstate0/;
7029
7030     my $storage_name = PVE::Storage::parse_volume_id($volid);
7031     my $scfg = $storecfg->{ids}->{$storage_name};
7032     die "could not find storage '$storage_name'\n" if !defined($scfg);
7033
7034     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7035         return 1;
7036     }
7037
7038     if ($volid =~ m/\.(qcow2|qed)$/){
7039         return 1;
7040     }
7041
7042     return;
7043 }
7044
7045 sub qga_check_running {
7046     my ($vmid, $nowarn) = @_;
7047
7048     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7049     if ($@) {
7050         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7051         return 0;
7052     }
7053     return 1;
7054 }
7055
7056 sub template_create {
7057     my ($vmid, $conf, $disk) = @_;
7058
7059     my $storecfg = PVE::Storage::config();
7060
7061     PVE::QemuConfig->foreach_volume($conf, sub {
7062         my ($ds, $drive) = @_;
7063
7064         return if drive_is_cdrom($drive);
7065         return if $disk && $ds ne $disk;
7066
7067         my $volid = $drive->{file};
7068         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7069
7070         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7071         $drive->{file} = $voliddst;
7072         $conf->{$ds} = print_drive($drive);
7073         PVE::QemuConfig->write_config($vmid, $conf);
7074     });
7075 }
7076
7077 sub convert_iscsi_path {
7078     my ($path) = @_;
7079
7080     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7081         my $portal = $1;
7082         my $target = $2;
7083         my $lun = $3;
7084
7085         my $initiator_name = get_initiator_name();
7086
7087         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7088                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7089     }
7090
7091     die "cannot convert iscsi path '$path', unkown format\n";
7092 }
7093
7094 sub qemu_img_convert {
7095     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7096
7097     my $storecfg = PVE::Storage::config();
7098     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7099     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7100
7101     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7102
7103     my $cachemode;
7104     my $src_path;
7105     my $src_is_iscsi = 0;
7106     my $src_format;
7107
7108     if ($src_storeid) {
7109         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7110         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7111         $src_format = qemu_img_format($src_scfg, $src_volname);
7112         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7113         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7114         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7115     } elsif (-f $src_volid) {
7116         $src_path = $src_volid;
7117         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7118             $src_format = $1;
7119         }
7120     }
7121
7122     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7123
7124     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7125     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7126     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7127     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7128
7129     my $cmd = [];
7130     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7131     push @$cmd, '-l', "snapshot.name=$snapname"
7132         if $snapname && $src_format && $src_format eq "qcow2";
7133     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7134     push @$cmd, '-T', $cachemode if defined($cachemode);
7135
7136     if ($src_is_iscsi) {
7137         push @$cmd, '--image-opts';
7138         $src_path = convert_iscsi_path($src_path);
7139     } elsif ($src_format) {
7140         push @$cmd, '-f', $src_format;
7141     }
7142
7143     if ($dst_is_iscsi) {
7144         push @$cmd, '--target-image-opts';
7145         $dst_path = convert_iscsi_path($dst_path);
7146     } else {
7147         push @$cmd, '-O', $dst_format;
7148     }
7149
7150     push @$cmd, $src_path;
7151
7152     if (!$dst_is_iscsi && $is_zero_initialized) {
7153         push @$cmd, "zeroinit:$dst_path";
7154     } else {
7155         push @$cmd, $dst_path;
7156     }
7157
7158     my $parser = sub {
7159         my $line = shift;
7160         if($line =~ m/\((\S+)\/100\%\)/){
7161             my $percent = $1;
7162             my $transferred = int($size * $percent / 100);
7163             my $total_h = render_bytes($size, 1);
7164             my $transferred_h = render_bytes($transferred, 1);
7165
7166             print "transferred $transferred_h of $total_h ($percent%)\n";
7167         }
7168
7169     };
7170
7171     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7172     my $err = $@;
7173     die "copy failed: $err" if $err;
7174 }
7175
7176 sub qemu_img_format {
7177     my ($scfg, $volname) = @_;
7178
7179     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7180         return $1;
7181     } else {
7182         return "raw";
7183     }
7184 }
7185
7186 sub qemu_drive_mirror {
7187     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7188
7189     $jobs = {} if !$jobs;
7190
7191     my $qemu_target;
7192     my $format;
7193     $jobs->{"drive-$drive"} = {};
7194
7195     if ($dst_volid =~ /^nbd:/) {
7196         $qemu_target = $dst_volid;
7197         $format = "nbd";
7198     } else {
7199         my $storecfg = PVE::Storage::config();
7200         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7201
7202         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7203
7204         $format = qemu_img_format($dst_scfg, $dst_volname);
7205
7206         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7207
7208         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7209     }
7210
7211     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7212     $opts->{format} = $format if $format;
7213
7214     if (defined($src_bitmap)) {
7215         $opts->{sync} = 'incremental';
7216         $opts->{bitmap} = $src_bitmap;
7217         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7218     }
7219
7220     if (defined($bwlimit)) {
7221         $opts->{speed} = $bwlimit * 1024;
7222         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7223     } else {
7224         print "drive mirror is starting for drive-$drive\n";
7225     }
7226
7227     # if a job already runs for this device we get an error, catch it for cleanup
7228     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7229     if (my $err = $@) {
7230         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7231         warn "$@\n" if $@;
7232         die "mirroring error: $err\n";
7233     }
7234
7235     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7236 }
7237
7238 # $completion can be either
7239 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7240 # 'cancel': wait until all jobs are ready, block-job-cancel them
7241 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7242 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7243 sub qemu_drive_mirror_monitor {
7244     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7245
7246     $completion //= 'complete';
7247     $op //= "mirror";
7248
7249     eval {
7250         my $err_complete = 0;
7251
7252         my $starttime = time ();
7253         while (1) {
7254             die "block job ('$op') timed out\n" if $err_complete > 300;
7255
7256             my $stats = mon_cmd($vmid, "query-block-jobs");
7257             my $ctime = time();
7258
7259             my $running_jobs = {};
7260             for my $stat (@$stats) {
7261                 next if $stat->{type} ne $op;
7262                 $running_jobs->{$stat->{device}} = $stat;
7263             }
7264
7265             my $readycounter = 0;
7266
7267             for my $job_id (sort keys %$jobs) {
7268                 my $job = $running_jobs->{$job_id};
7269
7270                 my $vanished = !defined($job);
7271                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7272                 if($complete || ($vanished && $completion eq 'auto')) {
7273                     print "$job_id: $op-job finished\n";
7274                     delete $jobs->{$job_id};
7275                     next;
7276                 }
7277
7278                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7279
7280                 my $busy = $job->{busy};
7281                 my $ready = $job->{ready};
7282                 if (my $total = $job->{len}) {
7283                     my $transferred = $job->{offset} || 0;
7284                     my $remaining = $total - $transferred;
7285                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7286
7287                     my $duration = $ctime - $starttime;
7288                     my $total_h = render_bytes($total, 1);
7289                     my $transferred_h = render_bytes($transferred, 1);
7290
7291                     my $status = sprintf(
7292                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7293                         render_duration($duration),
7294                     );
7295
7296                     if ($ready) {
7297                         if ($busy) {
7298                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7299                         } else {
7300                             $status .= ", ready";
7301                         }
7302                     }
7303                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7304                     $jobs->{$job_id}->{ready} = $ready;
7305                 }
7306
7307                 $readycounter++ if $job->{ready};
7308             }
7309
7310             last if scalar(keys %$jobs) == 0;
7311
7312             if ($readycounter == scalar(keys %$jobs)) {
7313                 print "all '$op' jobs are ready\n";
7314
7315                 # do the complete later (or has already been done)
7316                 last if $completion eq 'skip' || $completion eq 'auto';
7317
7318                 if ($vmiddst && $vmiddst != $vmid) {
7319                     my $agent_running = $qga && qga_check_running($vmid);
7320                     if ($agent_running) {
7321                         print "freeze filesystem\n";
7322                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7323                     } else {
7324                         print "suspend vm\n";
7325                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7326                     }
7327
7328                     # if we clone a disk for a new target vm, we don't switch the disk
7329                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7330
7331                     if ($agent_running) {
7332                         print "unfreeze filesystem\n";
7333                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7334                     } else {
7335                         print "resume vm\n";
7336                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7337                     }
7338
7339                     last;
7340                 } else {
7341
7342                     for my $job_id (sort keys %$jobs) {
7343                         # try to switch the disk if source and destination are on the same guest
7344                         print "$job_id: Completing block job_id...\n";
7345
7346                         my $op;
7347                         if ($completion eq 'complete') {
7348                             $op = 'block-job-complete';
7349                         } elsif ($completion eq 'cancel') {
7350                             $op = 'block-job-cancel';
7351                         } else {
7352                             die "invalid completion value: $completion\n";
7353                         }
7354                         eval { mon_cmd($vmid, $op, device => $job_id) };
7355                         if ($@ =~ m/cannot be completed/) {
7356                             print "$job_id: block job cannot be completed, trying again.\n";
7357                             $err_complete++;
7358                         }else {
7359                             print "$job_id: Completed successfully.\n";
7360                             $jobs->{$job_id}->{complete} = 1;
7361                         }
7362                     }
7363                 }
7364             }
7365             sleep 1;
7366         }
7367     };
7368     my $err = $@;
7369
7370     if ($err) {
7371         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7372         die "block job ($op) error: $err";
7373     }
7374 }
7375
7376 sub qemu_blockjobs_cancel {
7377     my ($vmid, $jobs) = @_;
7378
7379     foreach my $job (keys %$jobs) {
7380         print "$job: Cancelling block job\n";
7381         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7382         $jobs->{$job}->{cancel} = 1;
7383     }
7384
7385     while (1) {
7386         my $stats = mon_cmd($vmid, "query-block-jobs");
7387
7388         my $running_jobs = {};
7389         foreach my $stat (@$stats) {
7390             $running_jobs->{$stat->{device}} = $stat;
7391         }
7392
7393         foreach my $job (keys %$jobs) {
7394
7395             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7396                 print "$job: Done.\n";
7397                 delete $jobs->{$job};
7398             }
7399         }
7400
7401         last if scalar(keys %$jobs) == 0;
7402
7403         sleep 1;
7404     }
7405 }
7406
7407 sub clone_disk {
7408     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7409         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7410
7411     my $newvolid;
7412
7413     if (!$full) {
7414         print "create linked clone of drive $drivename ($drive->{file})\n";
7415         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7416         push @$newvollist, $newvolid;
7417     } else {
7418
7419         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7420         $storeid = $storage if $storage;
7421
7422         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7423
7424         print "create full clone of drive $drivename ($drive->{file})\n";
7425         my $name = undef;
7426         my $size = undef;
7427         if (drive_is_cloudinit($drive)) {
7428             $name = "vm-$newvmid-cloudinit";
7429             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7430             if ($scfg->{path}) {
7431                 $name .= ".$dst_format";
7432             }
7433             $snapname = undef;
7434             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7435         } elsif ($drivename eq 'efidisk0') {
7436             $size = get_efivars_size($conf);
7437         } elsif ($drivename eq 'tpmstate0') {
7438             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7439         } else {
7440             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7441         }
7442         $newvolid = PVE::Storage::vdisk_alloc(
7443             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7444         );
7445         push @$newvollist, $newvolid;
7446
7447         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7448
7449         if (drive_is_cloudinit($drive)) {
7450             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7451             # if this is the case, we have to complete any block-jobs still there from
7452             # previous drive-mirrors
7453             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7454                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7455             }
7456             goto no_data_clone;
7457         }
7458
7459         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7460         if (!$running || $snapname) {
7461             # TODO: handle bwlimits
7462             if ($drivename eq 'efidisk0') {
7463                 # the relevant data on the efidisk may be smaller than the source
7464                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7465                 # that is given by the OVMF_VARS.fd
7466                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7467                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7468
7469                 # better for Ceph if block size is not too small, see bug #3324
7470                 my $bs = 1024*1024;
7471
7472                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7473                     "if=$src_path", "of=$dst_path"]);
7474             } else {
7475                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7476             }
7477         } else {
7478
7479             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7480
7481             my $kvmver = get_running_qemu_version ($vmid);
7482             if (!min_version($kvmver, 2, 7)) {
7483                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7484                     if $drive->{iothread};
7485             }
7486
7487             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7488                 $completion, $qga, $bwlimit);
7489         }
7490     }
7491
7492 no_data_clone:
7493     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7494
7495     my $disk = $drive;
7496     $disk->{format} = undef;
7497     $disk->{file} = $newvolid;
7498     $disk->{size} = $size if defined($size);
7499
7500     return $disk;
7501 }
7502
7503 sub get_running_qemu_version {
7504     my ($vmid) = @_;
7505     my $res = mon_cmd($vmid, "query-version");
7506     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7507 }
7508
7509 sub qemu_use_old_bios_files {
7510     my ($machine_type) = @_;
7511
7512     return if !$machine_type;
7513
7514     my $use_old_bios_files = undef;
7515
7516     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7517         $machine_type = $1;
7518         $use_old_bios_files = 1;
7519     } else {
7520         my $version = extract_version($machine_type, kvm_user_version());
7521         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7522         # load new efi bios files on migration. So this hack is required to allow
7523         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7524         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7525         $use_old_bios_files = !min_version($version, 2, 4);
7526     }
7527
7528     return ($use_old_bios_files, $machine_type);
7529 }
7530
7531 sub get_efivars_size {
7532     my ($conf) = @_;
7533     my $arch = get_vm_arch($conf);
7534     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7535     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk);
7536     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7537     return -s $ovmf_vars;
7538 }
7539
7540 sub update_efidisk_size {
7541     my ($conf) = @_;
7542
7543     return if !defined($conf->{efidisk0});
7544
7545     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7546     $disk->{size} = get_efivars_size($conf);
7547     $conf->{efidisk0} = print_drive($disk);
7548
7549     return;
7550 }
7551
7552 sub update_tpmstate_size {
7553     my ($conf) = @_;
7554
7555     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7556     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7557     $conf->{tpmstate0} = print_drive($disk);
7558 }
7559
7560 sub create_efidisk($$$$$$) {
7561     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk) = @_;
7562
7563     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk);
7564     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7565
7566     my $vars_size_b = -s $ovmf_vars;
7567     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7568     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7569     PVE::Storage::activate_volumes($storecfg, [$volid]);
7570
7571     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7572     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7573
7574     return ($volid, $size/1024);
7575 }
7576
7577 sub vm_iothreads_list {
7578     my ($vmid) = @_;
7579
7580     my $res = mon_cmd($vmid, 'query-iothreads');
7581
7582     my $iothreads = {};
7583     foreach my $iothread (@$res) {
7584         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7585     }
7586
7587     return $iothreads;
7588 }
7589
7590 sub scsihw_infos {
7591     my ($conf, $drive) = @_;
7592
7593     my $maxdev = 0;
7594
7595     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7596         $maxdev = 7;
7597     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7598         $maxdev = 1;
7599     } else {
7600         $maxdev = 256;
7601     }
7602
7603     my $controller = int($drive->{index} / $maxdev);
7604     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7605         ? "virtioscsi"
7606         : "scsihw";
7607
7608     return ($maxdev, $controller, $controller_prefix);
7609 }
7610
7611 sub windows_version {
7612     my ($ostype) = @_;
7613
7614     return 0 if !$ostype;
7615
7616     my $winversion = 0;
7617
7618     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7619         $winversion = 5;
7620     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7621         $winversion = 6;
7622     } elsif ($ostype =~ m/^win(\d+)$/) {
7623         $winversion = $1;
7624     }
7625
7626     return $winversion;
7627 }
7628
7629 sub resolve_dst_disk_format {
7630         my ($storecfg, $storeid, $src_volname, $format) = @_;
7631         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7632
7633         if (!$format) {
7634             # if no target format is specified, use the source disk format as hint
7635             if ($src_volname) {
7636                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7637                 $format = qemu_img_format($scfg, $src_volname);
7638             } else {
7639                 return $defFormat;
7640             }
7641         }
7642
7643         # test if requested format is supported - else use default
7644         my $supported = grep { $_ eq $format } @$validFormats;
7645         $format = $defFormat if !$supported;
7646         return $format;
7647 }
7648
7649 # NOTE: if this logic changes, please update docs & possibly gui logic
7650 sub find_vmstate_storage {
7651     my ($conf, $storecfg) = @_;
7652
7653     # first, return storage from conf if set
7654     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7655
7656     my ($target, $shared, $local);
7657
7658     foreach_storage_used_by_vm($conf, sub {
7659         my ($sid) = @_;
7660         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7661         my $dst = $scfg->{shared} ? \$shared : \$local;
7662         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7663     });
7664
7665     # second, use shared storage where VM has at least one disk
7666     # third, use local storage where VM has at least one disk
7667     # fall back to local storage
7668     $target = $shared // $local // 'local';
7669
7670     return $target;
7671 }
7672
7673 sub generate_uuid {
7674     my ($uuid, $uuid_str);
7675     UUID::generate($uuid);
7676     UUID::unparse($uuid, $uuid_str);
7677     return $uuid_str;
7678 }
7679
7680 sub generate_smbios1_uuid {
7681     return "uuid=".generate_uuid();
7682 }
7683
7684 sub nbd_stop {
7685     my ($vmid) = @_;
7686
7687     mon_cmd($vmid, 'nbd-server-stop');
7688 }
7689
7690 sub create_reboot_request {
7691     my ($vmid) = @_;
7692     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7693         or die "failed to create reboot trigger file: $!\n";
7694     close($fh);
7695 }
7696
7697 sub clear_reboot_request {
7698     my ($vmid) = @_;
7699     my $path = "/run/qemu-server/$vmid.reboot";
7700     my $res = 0;
7701
7702     $res = unlink($path);
7703     die "could not remove reboot request for $vmid: $!"
7704         if !$res && $! != POSIX::ENOENT;
7705
7706     return $res;
7707 }
7708
7709 sub bootorder_from_legacy {
7710     my ($conf, $bootcfg) = @_;
7711
7712     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7713     my $bootindex_hash = {};
7714     my $i = 1;
7715     foreach my $o (split(//, $boot)) {
7716         $bootindex_hash->{$o} = $i*100;
7717         $i++;
7718     }
7719
7720     my $bootorder = {};
7721
7722     PVE::QemuConfig->foreach_volume($conf, sub {
7723         my ($ds, $drive) = @_;
7724
7725         if (drive_is_cdrom ($drive, 1)) {
7726             if ($bootindex_hash->{d}) {
7727                 $bootorder->{$ds} = $bootindex_hash->{d};
7728                 $bootindex_hash->{d} += 1;
7729             }
7730         } elsif ($bootindex_hash->{c}) {
7731             $bootorder->{$ds} = $bootindex_hash->{c}
7732                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7733             $bootindex_hash->{c} += 1;
7734         }
7735     });
7736
7737     if ($bootindex_hash->{n}) {
7738         for (my $i = 0; $i < $MAX_NETS; $i++) {
7739             my $netname = "net$i";
7740             next if !$conf->{$netname};
7741             $bootorder->{$netname} = $bootindex_hash->{n};
7742             $bootindex_hash->{n} += 1;
7743         }
7744     }
7745
7746     return $bootorder;
7747 }
7748
7749 # Generate default device list for 'boot: order=' property. Matches legacy
7750 # default boot order, but with explicit device names. This is important, since
7751 # the fallback for when neither 'order' nor the old format is specified relies
7752 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7753 sub get_default_bootdevices {
7754     my ($conf) = @_;
7755
7756     my @ret = ();
7757
7758     # harddisk
7759     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7760     push @ret, $first if $first;
7761
7762     # cdrom
7763     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7764     push @ret, $first if $first;
7765
7766     # network
7767     for (my $i = 0; $i < $MAX_NETS; $i++) {
7768         my $netname = "net$i";
7769         next if !$conf->{$netname};
7770         push @ret, $netname;
7771         last;
7772     }
7773
7774     return \@ret;
7775 }
7776
7777 sub device_bootorder {
7778     my ($conf) = @_;
7779
7780     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7781
7782     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7783
7784     my $bootorder = {};
7785     if (!defined($boot) || $boot->{legacy}) {
7786         $bootorder = bootorder_from_legacy($conf, $boot);
7787     } elsif ($boot->{order}) {
7788         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7789         for my $dev (PVE::Tools::split_list($boot->{order})) {
7790             $bootorder->{$dev} = $i++;
7791         }
7792     }
7793
7794     return $bootorder;
7795 }
7796
7797 sub register_qmeventd_handle {
7798     my ($vmid) = @_;
7799
7800     my $fh;
7801     my $peer = "/var/run/qmeventd.sock";
7802     my $count = 0;
7803
7804     for (;;) {
7805         $count++;
7806         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7807         last if $fh;
7808         if ($! != EINTR && $! != EAGAIN) {
7809             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7810         }
7811         if ($count > 4) {
7812             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7813               . "after $count retries\n";
7814         }
7815         usleep(25000);
7816     }
7817
7818     # send handshake to mark VM as backing up
7819     print $fh to_json({vzdump => {vmid => "$vmid"}});
7820
7821     # return handle to be closed later when inhibit is no longer required
7822     return $fh;
7823 }
7824
7825 # bash completion helper
7826
7827 sub complete_backup_archives {
7828     my ($cmdname, $pname, $cvalue) = @_;
7829
7830     my $cfg = PVE::Storage::config();
7831
7832     my $storeid;
7833
7834     if ($cvalue =~ m/^([^:]+):/) {
7835         $storeid = $1;
7836     }
7837
7838     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7839
7840     my $res = [];
7841     foreach my $id (keys %$data) {
7842         foreach my $item (@{$data->{$id}}) {
7843             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7844             push @$res, $item->{volid} if defined($item->{volid});
7845         }
7846     }
7847
7848     return $res;
7849 }
7850
7851 my $complete_vmid_full = sub {
7852     my ($running) = @_;
7853
7854     my $idlist = vmstatus();
7855
7856     my $res = [];
7857
7858     foreach my $id (keys %$idlist) {
7859         my $d = $idlist->{$id};
7860         if (defined($running)) {
7861             next if $d->{template};
7862             next if $running && $d->{status} ne 'running';
7863             next if !$running && $d->{status} eq 'running';
7864         }
7865         push @$res, $id;
7866
7867     }
7868     return $res;
7869 };
7870
7871 sub complete_vmid {
7872     return &$complete_vmid_full();
7873 }
7874
7875 sub complete_vmid_stopped {
7876     return &$complete_vmid_full(0);
7877 }
7878
7879 sub complete_vmid_running {
7880     return &$complete_vmid_full(1);
7881 }
7882
7883 sub complete_storage {
7884
7885     my $cfg = PVE::Storage::config();
7886     my $ids = $cfg->{ids};
7887
7888     my $res = [];
7889     foreach my $sid (keys %$ids) {
7890         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7891         next if !$ids->{$sid}->{content}->{images};
7892         push @$res, $sid;
7893     }
7894
7895     return $res;
7896 }
7897
7898 sub complete_migration_storage {
7899     my ($cmd, $param, $current_value, $all_args) = @_;
7900
7901     my $targetnode = @$all_args[1];
7902
7903     my $cfg = PVE::Storage::config();
7904     my $ids = $cfg->{ids};
7905
7906     my $res = [];
7907     foreach my $sid (keys %$ids) {
7908         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7909         next if !$ids->{$sid}->{content}->{images};
7910         push @$res, $sid;
7911     }
7912
7913     return $res;
7914 }
7915
7916 sub vm_is_paused {
7917     my ($vmid) = @_;
7918     my $qmpstatus = eval {
7919         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7920         mon_cmd($vmid, "query-status");
7921     };
7922     warn "$@\n" if $@;
7923     return $qmpstatus && $qmpstatus->{status} eq "paused";
7924 }
7925
7926 sub check_volume_storage_type {
7927     my ($storecfg, $vol) = @_;
7928
7929     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7930     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7931     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7932
7933     die "storage '$storeid' does not support content-type '$vtype'\n"
7934         if !$scfg->{content}->{$vtype};
7935
7936     return 1;
7937 }
7938
7939 1;