PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Shown in the web-interface VM's summary."
 369             ." This is saved as comment inside the configuration file.",
 370         maxLength => 1024 * 8,
 371     },
 372     ostype => {
 373         optional => 1,
 374         type => 'string',
 375         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 376         description => "Specify guest operating system.",
 377         verbose_description => <<EODESC,
 378 Specify guest operating system. This is used to enable special
 379 optimization/features for specific operating systems:
 380
 381 [horizontal]
 382 other;; unspecified OS
 383 wxp;; Microsoft Windows XP
 384 w2k;; Microsoft Windows 2000
 385 w2k3;; Microsoft Windows 2003
 386 w2k8;; Microsoft Windows 2008
 387 wvista;; Microsoft Windows Vista
 388 win7;; Microsoft Windows 7
 389 win8;; Microsoft Windows 8/2012/2012r2
 390 win10;; Microsoft Windows 10/2016/2019
 391 l24;; Linux 2.4 Kernel
 392 l26;; Linux 2.6 - 5.X Kernel
 393 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 394 EODESC
 395     },
 396     boot => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-boot',
 399         description => "Specify guest boot order. Use with 'order=', usage with"
 400                      . " no key or 'legacy=' is deprecated.",
 401     },
 402     bootdisk => {
 403         optional => 1,
 404         type => 'string', format => 'pve-qm-bootdisk',
 405         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 406         pattern => '(ide|sata|scsi|virtio)\d+',
 407     },
 408     smp => {
 409         optional => 1,
 410         type => 'integer',
 411         description => "The number of CPUs. Please use option -sockets instead.",
 412         minimum => 1,
 413         default => 1,
 414     },
 415     sockets => {
 416         optional => 1,
 417         type => 'integer',
 418         description => "The number of CPU sockets.",
 419         minimum => 1,
 420         default => 1,
 421     },
 422     cores => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "The number of cores per socket.",
 426         minimum => 1,
 427         default => 1,
 428     },
 429     numa => {
 430         optional => 1,
 431         type => 'boolean',
 432         description => "Enable/disable NUMA.",
 433         default => 0,
 434     },
 435     hugepages => {
 436         optional => 1,
 437         type => 'string',
 438         description => "Enable/disable hugepages memory.",
 439         enum => [qw(any 2 1024)],
 440     },
 441     keephugepages => {
 442         optional => 1,
 443         type => 'boolean',
 444         default => 0,
 445         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 446             ." after VM shutdown and can be used for subsequent starts.",
 447     },
 448     vcpus => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "Number of hotplugged vcpus.",
 452         minimum => 1,
 453         default => 0,
 454     },
 455     acpi => {
 456         optional => 1,
 457         type => 'boolean',
 458         description => "Enable/disable ACPI.",
 459         default => 1,
 460     },
 461     agent => {
 462         optional => 1,
 463         description => "Enable/disable Qemu GuestAgent and its properties.",
 464         type => 'string',
 465         format => $agent_fmt,
 466     },
 467     kvm => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable KVM hardware virtualization.",
 471         default => 1,
 472     },
 473     tdf => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable time drift fix.",
 477         default => 0,
 478     },
 479     localtime => {
 480         optional => 1,
 481         type => 'boolean',
 482         description => "Set the real time clock to local time. This is enabled by default if ostype"
 483             ." indicates a Microsoft OS.",
 484     },
 485     freeze => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 489     },
 490     vga => {
 491         optional => 1,
 492         type => 'string', format => $vga_fmt,
 493         description => "Configure the VGA hardware.",
 494         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 495             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 496             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 497             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 498             ." display server. For win* OS you can select how many independent displays you want,"
 499             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 500             ." using a serial device as terminal.",
 501     },
 502     watchdog => {
 503         optional => 1,
 504         type => 'string', format => 'pve-qm-watchdog',
 505         description => "Create a virtual hardware watchdog device.",
 506         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 507             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 508             ." else the watchdog will reset the guest (or execute the respective action specified)",
 509     },
 510     startdate => {
 511         optional => 1,
 512         type => 'string',
 513         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 514         description => "Set the initial date of the real time clock. Valid format for date are:"
 515             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 516         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 517         default => 'now',
 518     },
 519     startup =>  get_standard_option('pve-startup-order'),
 520     template => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Enable/disable Template.",
 524         default => 0,
 525     },
 526     args => {
 527         optional => 1,
 528         type => 'string',
 529         description => "Arbitrary arguments passed to kvm.",
 530         verbose_description => <<EODESCR,
 531 Arbitrary arguments passed to kvm, for example:
 532
 533 args: -no-reboot -no-hpet
 534
 535 NOTE: this option is for experts only.
 536 EODESCR
 537     },
 538     tablet => {
 539         optional => 1,
 540         type => 'boolean',
 541         default => 1,
 542         description => "Enable/disable the USB tablet device.",
 543         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 544             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 545             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 546             ." may consider disabling this to save some context switches. This is turned off by"
 547             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 548     },
 549     migrate_speed => {
 550         optional => 1,
 551         type => 'integer',
 552         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 553         minimum => 0,
 554         default => 0,
 555     },
 556     migrate_downtime => {
 557         optional => 1,
 558         type => 'number',
 559         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 560         minimum => 0,
 561         default => 0.1,
 562     },
 563     cdrom => {
 564         optional => 1,
 565         type => 'string', format => 'pve-qm-ide',
 566         typetext => '<volume>',
 567         description => "This is an alias for option -ide2",
 568     },
 569     cpu => {
 570         optional => 1,
 571         description => "Emulated CPU type.",
 572         type => 'string',
 573         format => 'pve-vm-cpu-conf',
 574     },
 575     parent => get_standard_option('pve-snapshot-name', {
 576         optional => 1,
 577         description => "Parent snapshot name. This is used internally, and should not be modified.",
 578     }),
 579     snaptime => {
 580         optional => 1,
 581         description => "Timestamp for snapshots.",
 582         type => 'integer',
 583         minimum => 0,
 584     },
 585     vmstate => {
 586         optional => 1,
 587         type => 'string', format => 'pve-volume-id',
 588         description => "Reference to a volume which stores the VM state. This is used internally"
 589             ." for snapshots.",
 590     },
 591     vmstatestorage => get_standard_option('pve-storage-id', {
 592         description => "Default storage for VM state volumes/files.",
 593         optional => 1,
 594     }),
 595     runningmachine => get_standard_option('pve-qemu-machine', {
 596         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 597             ." for snapshots.",
 598     }),
 599     runningcpu => {
 600         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 601             ." internally for snapshots.",
 602         optional => 1,
 603         type => 'string',
 604         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 605         format_description => 'QEMU -cpu parameter'
 606     },
 607     machine => get_standard_option('pve-qemu-machine'),
 608     arch => {
 609         description => "Virtual processor architecture. Defaults to the host.",
 610         optional => 1,
 611         type => 'string',
 612         enum => [qw(x86_64 aarch64)],
 613     },
 614     smbios1 => {
 615         description => "Specify SMBIOS type 1 fields.",
 616         type => 'string', format => 'pve-qm-smbios1',
 617         maxLength => 512,
 618         optional => 1,
 619     },
 620     protection => {
 621         optional => 1,
 622         type => 'boolean',
 623         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 624             ." remove disk operations.",
 625         default => 0,
 626     },
 627     bios => {
 628         optional => 1,
 629         type => 'string',
 630         enum => [ qw(seabios ovmf) ],
 631         description => "Select BIOS implementation.",
 632         default => 'seabios',
 633     },
 634     vmgenid => {
 635         type => 'string',
 636         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 637         format_description => 'UUID',
 638         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 639             ." to disable explicitly.",
 640         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 641             ." value identifier to the guest OS. This allows to notify the guest operating system"
 642             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 643             ." execution or creation from a template). The guest operating system notices the"
 644             ." change, and is then able to react as appropriate by marking its copies of"
 645             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 646             ."Note that auto-creation only works when done through API/CLI create or update methods"
 647             .", but not when manually editing the config file.",
 648         default => "1 (autogenerated)",
 649         optional => 1,
 650     },
 651     hookscript => {
 652         type => 'string',
 653         format => 'pve-volume-id',
 654         optional => 1,
 655         description => "Script that will be executed during various steps in the vms lifetime.",
 656     },
 657     ivshmem => {
 658         type => 'string',
 659         format => $ivshmem_fmt,
 660         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 661             ." the host.",
 662         optional => 1,
 663     },
 664     audio0 => {
 665         type => 'string',
 666         format => $audio_fmt,
 667         description => "Configure a audio device, useful in combination with QXL/Spice.",
 668         optional => 1
 669     },
 670     spice_enhancements => {
 671         type => 'string',
 672         format => $spice_enhancements_fmt,
 673         description => "Configure additional enhancements for SPICE.",
 674         optional => 1
 675     },
 676     tags => {
 677         type => 'string', format => 'pve-tag-list',
 678         description => 'Tags of the VM. This is only meta information.',
 679         optional => 1,
 680     },
 681     rng0 => {
 682         type => 'string',
 683         format => $rng_fmt,
 684         description => "Configure a VirtIO-based Random Number Generator.",
 685         optional => 1,
 686     },
 687 };
 688
 689 my $cicustom_fmt = {
 690     meta => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all meta data passed to the VM via"
 694             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 695         format => 'pve-volume-id',
 696         format_description => 'volume',
 697     },
 698     network => {
 699         type => 'string',
 700         optional => 1,
 701         description => 'Specify a custom file containing all network data passed to the VM via'
 702             .' cloud-init.',
 703         format => 'pve-volume-id',
 704         format_description => 'volume',
 705     },
 706     user => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all user data passed to the VM via'
 710             .' cloud-init.',
 711         format => 'pve-volume-id',
 712         format_description => 'volume',
 713     },
 714 };
 715 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 716
 717 my $confdesc_cloudinit = {
 718     citype => {
 719         optional => 1,
 720         type => 'string',
 721         description => 'Specifies the cloud-init configuration format. The default depends on the'
 722             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 723             .' and `configdrive2` for windows.',
 724         enum => ['configdrive2', 'nocloud', 'opennebula'],
 725     },
 726     ciuser => {
 727         optional => 1,
 728         type => 'string',
 729         description => "cloud-init: User name to change ssh keys and password for instead of the"
 730             ." image's configured default user.",
 731     },
 732     cipassword => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'cloud-init: Password to assign the user. Using this is generally not'
 736             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 737             .' support hashed passwords.',
 738     },
 739     cicustom => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'cloud-init: Specify custom files to replace the automatically generated'
 743             .' ones at start.',
 744         format => 'pve-qm-cicustom',
 745     },
 746     searchdomain => {
 747         optional => 1,
 748         type => 'string',
 749         description => "cloud-init: Sets DNS search domains for a container. Create will'
 750             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 751             .' are set.",
 752     },
 753     nameserver => {
 754         optional => 1,
 755         type => 'string', format => 'address-list',
 756         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 757             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 758             .' are set.",
 759     },
 760     sshkeys => {
 761         optional => 1,
 762         type => 'string',
 763         format => 'urlencoded',
 764         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 765     },
 766 };
 767
 768 # what about other qemu settings ?
 769 #cpu => 'string',
 770 #machine => 'string',
 771 #fda => 'file',
 772 #fdb => 'file',
 773 #mtdblock => 'file',
 774 #sd => 'file',
 775 #pflash => 'file',
 776 #snapshot => 'bool',
 777 #bootp => 'file',
 778 ##tftp => 'dir',
 779 ##smb => 'dir',
 780 #kernel => 'file',
 781 #append => 'string',
 782 #initrd => 'file',
 783 ##soundhw => 'string',
 784
 785 while (my ($k, $v) = each %$confdesc) {
 786     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 787 }
 788
 789 my $MAX_USB_DEVICES = 5;
 790 my $MAX_NETS = 32;
 791 my $MAX_SERIAL_PORTS = 4;
 792 my $MAX_PARALLEL_PORTS = 3;
 793 my $MAX_NUMA = 8;
 794
 795 my $numa_fmt = {
 796     cpus => {
 797         type => "string",
 798         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 799         description => "CPUs accessing this NUMA node.",
 800         format_description => "id[-id];...",
 801     },
 802     memory => {
 803         type => "number",
 804         description => "Amount of memory this NUMA node provides.",
 805         optional => 1,
 806     },
 807     hostnodes => {
 808         type => "string",
 809         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 810         description => "Host NUMA nodes to use.",
 811         format_description => "id[-id];...",
 812         optional => 1,
 813     },
 814     policy => {
 815         type => 'string',
 816         enum => [qw(preferred bind interleave)],
 817         description => "NUMA allocation policy.",
 818         optional => 1,
 819     },
 820 };
 821 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 822 my $numadesc = {
 823     optional => 1,
 824     type => 'string', format => $numa_fmt,
 825     description => "NUMA topology.",
 826 };
 827 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 828
 829 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 830     $confdesc->{"numa$i"} = $numadesc;
 831 }
 832
 833 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 834                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 835                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 836 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 837
 838 my $net_fmt_bridge_descr = <<__EOD__;
 839 Bridge to attach the network device to. The Proxmox VE standard bridge
 840 is called 'vmbr0'.
 841
 842 If you do not specify a bridge, we create a kvm user (NATed) network
 843 device, which provides DHCP and DNS services. The following addresses
 844 are used:
 845
 846  10.0.2.2   Gateway
 847  10.0.2.3   DNS Server
 848  10.0.2.4   SMB Server
 849
 850 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 851 __EOD__
 852
 853 my $net_fmt = {
 854     macaddr  => get_standard_option('mac-addr', {
 855         description => "MAC address. That address must be unique withing your network. This is"
 856             ." automatically generated if not specified.",
 857     }),
 858     model => {
 859         type => 'string',
 860         description => "Network Card Model. The 'virtio' model provides the best performance with"
 861             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 862             ." best to use 'e1000'.",
 863         enum => $nic_model_list,
 864         default_key => 1,
 865     },
 866     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 867     bridge => {
 868         type => 'string',
 869         description => $net_fmt_bridge_descr,
 870         format_description => 'bridge',
 871         pattern => '[-_.\w\d]+',
 872         optional => 1,
 873     },
 874     queues => {
 875         type => 'integer',
 876         minimum => 0, maximum => 16,
 877         description => 'Number of packet queues to be used on the device.',
 878         optional => 1,
 879     },
 880     rate => {
 881         type => 'number',
 882         minimum => 0,
 883         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 884         optional => 1,
 885     },
 886     tag => {
 887         type => 'integer',
 888         minimum => 1, maximum => 4094,
 889         description => 'VLAN tag to apply to packets on this interface.',
 890         optional => 1,
 891     },
 892     trunks => {
 893         type => 'string',
 894         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 895         description => 'VLAN trunks to pass through this interface.',
 896         format_description => 'vlanid[;vlanid...]',
 897         optional => 1,
 898     },
 899     firewall => {
 900         type => 'boolean',
 901         description => 'Whether this interface should be protected by the firewall.',
 902         optional => 1,
 903     },
 904     link_down => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be disconnected (like pulling the plug).',
 907         optional => 1,
 908     },
 909     mtu => {
 910         type => 'integer',
 911         minimum => 1, maximum => 65520,
 912         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 913         optional => 1,
 914     },
 915 };
 916
 917 my $netdesc = {
 918     optional => 1,
 919     type => 'string', format => $net_fmt,
 920     description => "Specify network devices.",
 921 };
 922
 923 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 924
 925 my $ipconfig_fmt = {
 926     ip => {
 927         type => 'string',
 928         format => 'pve-ipv4-config',
 929         format_description => 'IPv4Format/CIDR',
 930         description => 'IPv4 address in CIDR format.',
 931         optional => 1,
 932         default => 'dhcp',
 933     },
 934     gw => {
 935         type => 'string',
 936         format => 'ipv4',
 937         format_description => 'GatewayIPv4',
 938         description => 'Default gateway for IPv4 traffic.',
 939         optional => 1,
 940         requires => 'ip',
 941     },
 942     ip6 => {
 943         type => 'string',
 944         format => 'pve-ipv6-config',
 945         format_description => 'IPv6Format/CIDR',
 946         description => 'IPv6 address in CIDR format.',
 947         optional => 1,
 948         default => 'dhcp',
 949     },
 950     gw6 => {
 951         type => 'string',
 952         format => 'ipv6',
 953         format_description => 'GatewayIPv6',
 954         description => 'Default gateway for IPv6 traffic.',
 955         optional => 1,
 956         requires => 'ip6',
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 960 my $ipconfigdesc = {
 961     optional => 1,
 962     type => 'string', format => 'pve-qm-ipconfig',
 963     description => <<'EODESCR',
 964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 965
 966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 967
 968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 969 gateway should be provided.
 970 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 971 cloud-init 19.4 or newer.
 972
 973 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 974 dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $usb_fmt = {
1006     host => {
1007         default_key => 1,
1008         type => 'string', format => 'pve-qm-usb-device',
1009         format_description => 'HOSTUSBDEVICE|spice',
1010         description => <<EODESCR,
1011 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1012
1013  'bus-port(.port)*' (decimal numbers) or
1014  'vendor_id:product_id' (hexadeciaml numbers) or
1015  'spice'
1016
1017 You can use the 'lsusb -t' command to list existing usb devices.
1018
1019 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1020 machines - use with special care.
1021
1022 The value 'spice' can be used to add a usb redirection devices for spice.
1023 EODESCR
1024     },
1025     usb3 => {
1026         optional => 1,
1027         type => 'boolean',
1028         description => "Specifies whether if given host option is a USB3 device or port.",
1029         default => 0,
1030     },
1031 };
1032
1033 my $usbdesc = {
1034     optional => 1,
1035     type => 'string', format => $usb_fmt,
1036     description => "Configure an USB device (n is 0 to 4).",
1037 };
1038 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1039
1040 my $serialdesc = {
1041         optional => 1,
1042         type => 'string',
1043         pattern => '(/dev/.+|socket)',
1044         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1045         verbose_description =>  <<EODESCR,
1046 Create a serial device inside the VM (n is 0 to 3), and pass through a
1047 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1048 host side (use 'qm terminal' to open a terminal connection).
1049
1050 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1051 use with special care.
1052
1053 CAUTION: Experimental! User reported problems with this option.
1054 EODESCR
1055 };
1056
1057 my $paralleldesc= {
1058         optional => 1,
1059         type => 'string',
1060         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1061         description =>  "Map host parallel devices (n is 0 to 2).",
1062         verbose_description =>  <<EODESCR,
1063 Map host parallel devices (n is 0 to 2).
1064
1065 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1066 machines - use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1073     $confdesc->{"parallel$i"} = $paralleldesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1077     $confdesc->{"serial$i"} = $serialdesc;
1078 }
1079
1080 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1081     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1082 }
1083
1084 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1085     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1086 }
1087
1088 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1089     $confdesc->{"usb$i"} = $usbdesc;
1090 }
1091
1092 my $boot_fmt = {
1093     legacy => {
1094         optional => 1,
1095         default_key => 1,
1096         type => 'string',
1097         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1098                      . " Deprecated, use 'order=' instead.",
1099         pattern => '[acdn]{1,4}',
1100         format_description => "[acdn]{1,4}",
1101
1102         # note: this is also the fallback if boot: is not given at all
1103         default => 'cdn',
1104     },
1105     order => {
1106         optional => 1,
1107         type => 'string',
1108         format => 'pve-qm-bootdev-list',
1109         format_description => "device[;device...]",
1110         description => <<EODESC,
1111 The guest will attempt to boot from devices in the order they appear here.
1112
1113 Disks, optical drives and passed-through storage USB devices will be directly
1114 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1115 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1116
1117 Note that only devices in this list will be marked as bootable and thus loaded
1118 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1119 (e.g. software-raid), you need to specify all of them here.
1120
1121 Overrides the deprecated 'legacy=[acdn]*' value when given.
1122 EODESC
1123     },
1124 };
1125 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1126
1127 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1128 sub verify_bootdev {
1129     my ($dev, $noerr) = @_;
1130
1131     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1132
1133     my $check = sub {
1134         my ($base) = @_;
1135         return 0 if $dev !~ m/^$base\d+$/;
1136         return 0 if !$confdesc->{$dev};
1137         return 1;
1138     };
1139
1140     return $dev if $check->("net");
1141     return $dev if $check->("usb");
1142     return $dev if $check->("hostpci");
1143
1144     return if $noerr;
1145     die "invalid boot device '$dev'\n";
1146 }
1147
1148 sub print_bootorder {
1149     my ($devs) = @_;
1150     return "" if !@$devs;
1151     my $data = { order => join(';', @$devs) };
1152     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1153 }
1154
1155 my $kvm_api_version = 0;
1156
1157 sub kvm_version {
1158     return $kvm_api_version if $kvm_api_version;
1159
1160     open my $fh, '<', '/dev/kvm' or return;
1161
1162     # 0xae00 => KVM_GET_API_VERSION
1163     $kvm_api_version = ioctl($fh, 0xae00, 0);
1164     close($fh);
1165
1166     return $kvm_api_version;
1167 }
1168
1169 my $kvm_user_version = {};
1170 my $kvm_mtime = {};
1171
1172 sub kvm_user_version {
1173     my ($binary) = @_;
1174
1175     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1176     my $st = stat($binary);
1177
1178     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180         $cachedmtime == $st->mtime;
1181
1182     $kvm_user_version->{$binary} = 'unknown';
1183     $kvm_mtime->{$binary} = $st->mtime;
1184
1185     my $code = sub {
1186         my $line = shift;
1187         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188             $kvm_user_version->{$binary} = $2;
1189         }
1190     };
1191
1192     eval { run_command([$binary, '--version'], outfunc => $code); };
1193     warn $@ if $@;
1194
1195     return $kvm_user_version->{$binary};
1196
1197 }
1198 my sub extract_version {
1199     my ($machine_type, $version) = @_;
1200     $version = kvm_user_version() if !defined($version);
1201     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1202 }
1203
1204 sub kernel_has_vhost_net {
1205     return -c '/dev/vhost-net';
1206 }
1207
1208 sub option_exists {
1209     my $key = shift;
1210     return defined($confdesc->{$key});
1211 }
1212
1213 my $cdrom_path;
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241      if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1286             if !$vtype;
1287         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1288         verify_media_type($opt, $vtype, $drive->{media});
1289         $drive->{file} = $volid;
1290     }
1291
1292     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1293 }
1294
1295 sub parse_hotplug_features {
1296     my ($data) = @_;
1297
1298     my $res = {};
1299
1300     return $res if $data eq '0';
1301
1302     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1303
1304     foreach my $feature (PVE::Tools::split_list($data)) {
1305         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306             $res->{$1} = 1;
1307         } else {
1308             die "invalid hotplug feature '$feature'\n";
1309         }
1310     }
1311     return $res;
1312 }
1313
1314 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1315 sub pve_verify_hotplug_features {
1316     my ($value, $noerr) = @_;
1317
1318     return $value if parse_hotplug_features($value);
1319
1320     return if $noerr;
1321
1322     die "unable to parse hotplug option\n";
1323 }
1324
1325 sub scsi_inquiry {
1326     my($fh, $noerr) = @_;
1327
1328     my $SG_IO = 0x2285;
1329     my $SG_GET_VERSION_NUM = 0x2282;
1330
1331     my $versionbuf = "\x00" x 8;
1332     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333     if (!$ret) {
1334         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335         return;
1336     }
1337     my $version = unpack("I", $versionbuf);
1338     if ($version < 30000) {
1339         die "scsi generic interface too old\n"  if !$noerr;
1340         return;
1341     }
1342
1343     my $buf = "\x00" x 36;
1344     my $sensebuf = "\x00" x 8;
1345     my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347     # see /usr/include/scsi/sg.h
1348     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351                       length($sensebuf), 0, length($buf), $buf,
1352                       $cmd, $sensebuf, 6000);
1353
1354     $ret = ioctl($fh, $SG_IO, $packet);
1355     if (!$ret) {
1356         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357         return;
1358     }
1359
1360     my @res = unpack($sg_io_hdr_t, $packet);
1361     if ($res[17] || $res[18]) {
1362         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363         return;
1364     }
1365
1366     my $res = {};
1367     (my $byte0, my $byte1, $res->{vendor},
1368      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370     $res->{removable} = $byte1 & 128 ? 1 : 0;
1371     $res->{type} = $byte0 & 31;
1372
1373     return $res;
1374 }
1375
1376 sub path_is_scsi {
1377     my ($path) = @_;
1378
1379     my $fh = IO::File->new("+<$path") || return;
1380     my $res = scsi_inquiry($fh, 1);
1381     close($fh);
1382
1383     return $res;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf, $arch) = @_;
1388
1389     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus;
1393     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1394         $usbbus = 'ehci';
1395     } else {
1396         $usbbus = 'uhci';
1397     }
1398
1399     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400 }
1401
1402 sub print_keyboarddevice_full {
1403     my ($conf, $arch, $machine) = @_;
1404
1405     return if $arch ne 'aarch64';
1406
1407     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408 }
1409
1410 my sub get_drive_id {
1411     my ($drive) = @_;
1412     return "$drive->{interface}$drive->{index}";
1413 }
1414
1415 sub print_drivedevice_full {
1416     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1417
1418     my $device = '';
1419     my $maxdev = 0;
1420
1421     my $drive_id = get_drive_id($drive);
1422     if ($drive->{interface} eq 'virtio') {
1423         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1424         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1425         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1426     } elsif ($drive->{interface} eq 'scsi') {
1427
1428         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1429         my $unit = $drive->{index} % $maxdev;
1430         my $devicetype = 'hd';
1431         my $path = '';
1432         if (drive_is_cdrom($drive)) {
1433             $devicetype = 'cd';
1434         } else {
1435             if ($drive->{file} =~ m|^/|) {
1436                 $path = $drive->{file};
1437                 if (my $info = path_is_scsi($path)) {
1438                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1439                         $devicetype = 'block';
1440                     } elsif ($info->{type} == 1) { # tape
1441                         $devicetype = 'generic';
1442                     }
1443                 }
1444             } else {
1445                  $path = PVE::Storage::path($storecfg, $drive->{file});
1446             }
1447
1448             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1449             my $version = extract_version($machine_type, kvm_user_version());
1450             if ($path =~ m/^iscsi\:\/\// &&
1451                !min_version($version, 4, 1)) {
1452                 $devicetype = 'generic';
1453             }
1454         }
1455
1456         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1458         } else {
1459             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1460                 .",lun=$drive->{index}";
1461         }
1462         $device .= ",drive=drive-$drive_id,id=$drive_id";
1463
1464         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1465             $device .= ",rotation_rate=1";
1466         }
1467         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1468
1469     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1470         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1471         my $controller = int($drive->{index} / $maxdev);
1472         my $unit = $drive->{index} % $maxdev;
1473         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1474
1475         $device = "ide-$devicetype";
1476         if ($drive->{interface} eq 'ide') {
1477             $device .= ",bus=ide.$controller,unit=$unit";
1478         } else {
1479             $device .= ",bus=ahci$controller.$unit";
1480         }
1481         $device .= ",drive=drive-$drive_id,id=$drive_id";
1482
1483         if ($devicetype eq 'hd') {
1484             if (my $model = $drive->{model}) {
1485                 $model = URI::Escape::uri_unescape($model);
1486                 $device .= ",model=$model";
1487             }
1488             if ($drive->{ssd}) {
1489                 $device .= ",rotation_rate=1";
1490             }
1491         }
1492         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1493     } elsif ($drive->{interface} eq 'usb') {
1494         die "implement me";
1495         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1496     } else {
1497         die "unsupported interface type";
1498     }
1499
1500     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1501
1502     if (my $serial = $drive->{serial}) {
1503         $serial = URI::Escape::uri_unescape($serial);
1504         $device .= ",serial=$serial";
1505     }
1506
1507
1508     return $device;
1509 }
1510
1511 sub get_initiator_name {
1512     my $initiator;
1513
1514     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1515     while (defined(my $line = <$fh>)) {
1516         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1517         $initiator = $1;
1518         last;
1519     }
1520     $fh->close();
1521
1522     return $initiator;
1523 }
1524
1525 sub print_drive_commandline_full {
1526     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1527
1528     my $path;
1529     my $volid = $drive->{file};
1530     my $format = $drive->{format};
1531     my $drive_id = get_drive_id($drive);
1532
1533     if (drive_is_cdrom($drive)) {
1534         $path = get_iso_path($storecfg, $vmid, $volid);
1535         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1536     } else {
1537         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1538         if ($storeid) {
1539             $path = PVE::Storage::path($storecfg, $volid);
1540             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1541             $format //= qemu_img_format($scfg, $volname);
1542         } else {
1543             $path = $volid;
1544             $format //= "raw";
1545         }
1546    }
1547
1548    my $is_rbd = $path =~ m/^rbd:/;
1549
1550     my $opts = '';
1551     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1552     foreach my $o (@qemu_drive_options) {
1553         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1554     }
1555
1556     # snapshot only accepts on|off
1557     if (defined($drive->{snapshot})) {
1558         my $v = $drive->{snapshot} ? 'on' : 'off';
1559         $opts .= ",snapshot=$v";
1560     }
1561
1562     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1563         my ($dir, $qmpname) = @$type;
1564         if (my $v = $drive->{"mbps$dir"}) {
1565             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1566         }
1567         if (my $v = $drive->{"mbps${dir}_max"}) {
1568             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1569         }
1570         if (my $v = $drive->{"bps${dir}_max_length"}) {
1571             $opts .= ",throttling.bps$qmpname-max-length=$v";
1572         }
1573         if (my $v = $drive->{"iops${dir}"}) {
1574             $opts .= ",throttling.iops$qmpname=$v";
1575         }
1576         if (my $v = $drive->{"iops${dir}_max"}) {
1577             $opts .= ",throttling.iops$qmpname-max=$v";
1578         }
1579         if (my $v = $drive->{"iops${dir}_max_length"}) {
1580             $opts .= ",throttling.iops$qmpname-max-length=$v";
1581         }
1582     }
1583
1584     if ($pbs_name) {
1585         $format = "rbd" if $is_rbd;
1586         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1587             if !$format;
1588         $opts .= ",format=alloc-track,file.driver=$format";
1589     } elsif ($format) {
1590         $opts .= ",format=$format";
1591     }
1592
1593     my $cache_direct = 0;
1594
1595     if (my $cache = $drive->{cache}) {
1596         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1597     } elsif (!drive_is_cdrom($drive)) {
1598         $opts .= ",cache=none";
1599         $cache_direct = 1;
1600     }
1601
1602     if (!$drive->{aio}) {
1603         if ($io_uring) {
1604             # io_uring supports all cache modes
1605             $opts .= ",aio=io_uring";
1606         } else {
1607             # aio native works only with O_DIRECT
1608             if($cache_direct) {
1609                 $opts .= ",aio=native";
1610             } else {
1611                 $opts .= ",aio=threads";
1612             }
1613         }
1614     }
1615
1616     if (!drive_is_cdrom($drive)) {
1617         my $detectzeroes;
1618         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1619             $detectzeroes = 'off';
1620         } elsif ($drive->{discard}) {
1621             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1622         } else {
1623             # This used to be our default with discard not being specified:
1624             $detectzeroes = 'on';
1625         }
1626
1627         # note: 'detect-zeroes' works per blockdev and we want it to persist
1628         # after the alloc-track is removed, so put it on 'file' directly
1629         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1630         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1631     }
1632
1633     if ($pbs_name) {
1634         $opts .= ",backing=$pbs_name";
1635         $opts .= ",auto-remove=on";
1636     }
1637
1638     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1639     my $file_param = "file";
1640     if ($pbs_name) {
1641         # non-rbd drivers require the underlying file to be a seperate block
1642         # node, so add a second .file indirection
1643         $file_param .= ".file" if !$is_rbd;
1644         $file_param .= ".filename";
1645     }
1646     my $pathinfo = $path ? "$file_param=$path," : '';
1647
1648     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1649 }
1650
1651 sub print_pbs_blockdev {
1652     my ($pbs_conf, $pbs_name) = @_;
1653     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1654     $blockdev .= ",repository=$pbs_conf->{repository}";
1655     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1656     $blockdev .= ",archive=$pbs_conf->{archive}";
1657     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1658     return $blockdev;
1659 }
1660
1661 sub print_netdevice_full {
1662     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1663
1664     my $device = $net->{model};
1665     if ($net->{model} eq 'virtio') {
1666          $device = 'virtio-net-pci';
1667      };
1668
1669     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1670     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1671     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1672         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1673         # and out of each queue plus one config interrupt and control vector queue
1674         my $vectors = $net->{queues} * 2 + 2;
1675         $tmpstr .= ",vectors=$vectors,mq=on";
1676     }
1677     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1678
1679     if (my $mtu = $net->{mtu}) {
1680         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1681             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1682             if ($mtu == 1) {
1683                  $mtu = $bridge_mtu;
1684             } elsif ($mtu < 576) {
1685                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1686             } elsif ($mtu > $bridge_mtu) {
1687                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1688             }
1689             $tmpstr .= ",host_mtu=$mtu";
1690         } else {
1691             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1692         }
1693     }
1694
1695     if ($use_old_bios_files) {
1696         my $romfile;
1697         if ($device eq 'virtio-net-pci') {
1698             $romfile = 'pxe-virtio.rom';
1699         } elsif ($device eq 'e1000') {
1700             $romfile = 'pxe-e1000.rom';
1701         } elsif ($device eq 'ne2k') {
1702             $romfile = 'pxe-ne2k_pci.rom';
1703         } elsif ($device eq 'pcnet') {
1704             $romfile = 'pxe-pcnet.rom';
1705         } elsif ($device eq 'rtl8139') {
1706             $romfile = 'pxe-rtl8139.rom';
1707         }
1708         $tmpstr .= ",romfile=$romfile" if $romfile;
1709     }
1710
1711     return $tmpstr;
1712 }
1713
1714 sub print_netdev_full {
1715     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1716
1717     my $i = '';
1718     if ($netid =~ m/^net(\d+)$/) {
1719         $i = int($1);
1720     }
1721
1722     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1723
1724     my $ifname = "tap${vmid}i$i";
1725
1726     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1727     die "interface name '$ifname' is too long (max 15 character)\n"
1728         if length($ifname) >= 16;
1729
1730     my $vhostparam = '';
1731     if (is_native($arch)) {
1732         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1733     }
1734
1735     my $vmname = $conf->{name} || "vm$vmid";
1736
1737     my $netdev = "";
1738     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1739
1740     if ($net->{bridge}) {
1741         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1742             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1743     } else {
1744         $netdev = "type=user,id=$netid,hostname=$vmname";
1745     }
1746
1747     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1748
1749     return $netdev;
1750 }
1751
1752 my $vga_map = {
1753     'cirrus' => 'cirrus-vga',
1754     'std' => 'VGA',
1755     'vmware' => 'vmware-svga',
1756     'virtio' => 'virtio-vga',
1757 };
1758
1759 sub print_vga_device {
1760     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1761
1762     my $type = $vga_map->{$vga->{type}};
1763     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1764         $type = 'virtio-gpu';
1765     }
1766     my $vgamem_mb = $vga->{memory};
1767
1768     my $max_outputs = '';
1769     if ($qxlnum) {
1770         $type = $id ? 'qxl' : 'qxl-vga';
1771
1772         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1773             # set max outputs so linux can have up to 4 qxl displays with one device
1774             if (min_version($machine_version, 4, 1)) {
1775                 $max_outputs = ",max_outputs=4";
1776             }
1777         }
1778     }
1779
1780     die "no devicetype for $vga->{type}\n" if !$type;
1781
1782     my $memory = "";
1783     if ($vgamem_mb) {
1784         if ($vga->{type} eq 'virtio') {
1785             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1786             $memory = ",max_hostmem=$bytes";
1787         } elsif ($qxlnum) {
1788             # from https://www.spice-space.org/multiple-monitors.html
1789             $memory = ",vgamem_mb=$vga->{memory}";
1790             my $ram = $vgamem_mb * 4;
1791             my $vram = $vgamem_mb * 2;
1792             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1793         } else {
1794             $memory = ",vgamem_mb=$vga->{memory}";
1795         }
1796     } elsif ($qxlnum && $id) {
1797         $memory = ",ram_size=67108864,vram_size=33554432";
1798     }
1799
1800     my $edidoff = "";
1801     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1802         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1803     }
1804
1805     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1806     my $vgaid = "vga" . ($id // '');
1807     my $pciaddr;
1808
1809     if ($q35 && $vgaid eq 'vga') {
1810         # the first display uses pcie.0 bus on q35 machines
1811         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1812     } else {
1813         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1814     }
1815
1816     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1817 }
1818
1819 sub parse_number_sets {
1820     my ($set) = @_;
1821     my $res = [];
1822     foreach my $part (split(/;/, $set)) {
1823         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1824             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1825             push @$res, [ $1, $2 ];
1826         } else {
1827             die "invalid range: $part\n";
1828         }
1829     }
1830     return $res;
1831 }
1832
1833 sub parse_numa {
1834     my ($data) = @_;
1835
1836     my $res = parse_property_string($numa_fmt, $data);
1837     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1838     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1839     return $res;
1840 }
1841
1842 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1843 sub parse_net {
1844     my ($data) = @_;
1845
1846     my $res = eval { parse_property_string($net_fmt, $data) };
1847     if ($@) {
1848         warn $@;
1849         return;
1850     }
1851     if (!defined($res->{macaddr})) {
1852         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1853         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1854     }
1855     return $res;
1856 }
1857
1858 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1859 sub parse_ipconfig {
1860     my ($data) = @_;
1861
1862     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1863     if ($@) {
1864         warn $@;
1865         return;
1866     }
1867
1868     if ($res->{gw} && !$res->{ip}) {
1869         warn 'gateway specified without specifying an IP address';
1870         return;
1871     }
1872     if ($res->{gw6} && !$res->{ip6}) {
1873         warn 'IPv6 gateway specified without specifying an IPv6 address';
1874         return;
1875     }
1876     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1877         warn 'gateway specified together with DHCP';
1878         return;
1879     }
1880     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1881         # gw6 + auto/dhcp
1882         warn "IPv6 gateway specified together with $res->{ip6} address";
1883         return;
1884     }
1885
1886     if (!$res->{ip} && !$res->{ip6}) {
1887         return { ip => 'dhcp', ip6 => 'dhcp' };
1888     }
1889
1890     return $res;
1891 }
1892
1893 sub print_net {
1894     my $net = shift;
1895
1896     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1897 }
1898
1899 sub add_random_macs {
1900     my ($settings) = @_;
1901
1902     foreach my $opt (keys %$settings) {
1903         next if $opt !~ m/^net(\d+)$/;
1904         my $net = parse_net($settings->{$opt});
1905         next if !$net;
1906         $settings->{$opt} = print_net($net);
1907     }
1908 }
1909
1910 sub vm_is_volid_owner {
1911     my ($storecfg, $vmid, $volid) = @_;
1912
1913     if ($volid !~  m|^/|) {
1914         my ($path, $owner);
1915         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1916         if ($owner && ($owner == $vmid)) {
1917             return 1;
1918         }
1919     }
1920
1921     return;
1922 }
1923
1924 sub vmconfig_register_unused_drive {
1925     my ($storecfg, $vmid, $conf, $drive) = @_;
1926
1927     if (drive_is_cloudinit($drive)) {
1928         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1929         warn $@ if $@;
1930     } elsif (!drive_is_cdrom($drive)) {
1931         my $volid = $drive->{file};
1932         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1933             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1934         }
1935     }
1936 }
1937
1938 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1939 my $smbios1_fmt = {
1940     uuid => {
1941         type => 'string',
1942         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1943         format_description => 'UUID',
1944         description => "Set SMBIOS1 UUID.",
1945         optional => 1,
1946     },
1947     version => {
1948         type => 'string',
1949         pattern => '[A-Za-z0-9+\/]+={0,2}',
1950         format_description => 'Base64 encoded string',
1951         description => "Set SMBIOS1 version.",
1952         optional => 1,
1953     },
1954     serial => {
1955         type => 'string',
1956         pattern => '[A-Za-z0-9+\/]+={0,2}',
1957         format_description => 'Base64 encoded string',
1958         description => "Set SMBIOS1 serial number.",
1959         optional => 1,
1960     },
1961     manufacturer => {
1962         type => 'string',
1963         pattern => '[A-Za-z0-9+\/]+={0,2}',
1964         format_description => 'Base64 encoded string',
1965         description => "Set SMBIOS1 manufacturer.",
1966         optional => 1,
1967     },
1968     product => {
1969         type => 'string',
1970         pattern => '[A-Za-z0-9+\/]+={0,2}',
1971         format_description => 'Base64 encoded string',
1972         description => "Set SMBIOS1 product ID.",
1973         optional => 1,
1974     },
1975     sku => {
1976         type => 'string',
1977         pattern => '[A-Za-z0-9+\/]+={0,2}',
1978         format_description => 'Base64 encoded string',
1979         description => "Set SMBIOS1 SKU string.",
1980         optional => 1,
1981     },
1982     family => {
1983         type => 'string',
1984         pattern => '[A-Za-z0-9+\/]+={0,2}',
1985         format_description => 'Base64 encoded string',
1986         description => "Set SMBIOS1 family string.",
1987         optional => 1,
1988     },
1989     base64 => {
1990         type => 'boolean',
1991         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1992         optional => 1,
1993     },
1994 };
1995
1996 sub parse_smbios1 {
1997     my ($data) = @_;
1998
1999     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2000     warn $@ if $@;
2001     return $res;
2002 }
2003
2004 sub print_smbios1 {
2005     my ($smbios1) = @_;
2006     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2010
2011 sub parse_watchdog {
2012     my ($value) = @_;
2013
2014     return if !$value;
2015
2016     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2017     warn $@ if $@;
2018     return $res;
2019 }
2020
2021 sub parse_guest_agent {
2022     my ($conf) = @_;
2023
2024     return {} if !defined($conf->{agent});
2025
2026     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2027     warn $@ if $@;
2028
2029     # if the agent is disabled ignore the other potentially set properties
2030     return {} if !$res->{enabled};
2031     return $res;
2032 }
2033
2034 sub get_qga_key {
2035     my ($conf, $key) = @_;
2036     return undef if !defined($conf->{agent});
2037
2038     my $agent = parse_guest_agent($conf);
2039     return $agent->{$key};
2040 }
2041
2042 sub parse_vga {
2043     my ($value) = @_;
2044
2045     return {} if !$value;
2046     my $res = eval { parse_property_string($vga_fmt, $value) };
2047     warn $@ if $@;
2048     return $res;
2049 }
2050
2051 sub parse_rng {
2052     my ($value) = @_;
2053
2054     return if !$value;
2055
2056     my $res = eval { parse_property_string($rng_fmt, $value) };
2057     warn $@ if $@;
2058     return $res;
2059 }
2060
2061 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2062 sub verify_usb_device {
2063     my ($value, $noerr) = @_;
2064
2065     return $value if parse_usb_device($value);
2066
2067     return if $noerr;
2068
2069     die "unable to parse usb device\n";
2070 }
2071
2072 # add JSON properties for create and set function
2073 sub json_config_properties {
2074     my $prop = shift;
2075
2076     foreach my $opt (keys %$confdesc) {
2077         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2078             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2079         $prop->{$opt} = $confdesc->{$opt};
2080     }
2081
2082     return $prop;
2083 }
2084
2085 # return copy of $confdesc_cloudinit to generate documentation
2086 sub cloudinit_config_properties {
2087
2088     return dclone($confdesc_cloudinit);
2089 }
2090
2091 sub check_type {
2092     my ($key, $value) = @_;
2093
2094     die "unknown setting '$key'\n" if !$confdesc->{$key};
2095
2096     my $type = $confdesc->{$key}->{type};
2097
2098     if (!defined($value)) {
2099         die "got undefined value\n";
2100     }
2101
2102     if ($value =~ m/[\n\r]/) {
2103         die "property contains a line feed\n";
2104     }
2105
2106     if ($type eq 'boolean') {
2107         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2108         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2109         die "type check ('boolean') failed - got '$value'\n";
2110     } elsif ($type eq 'integer') {
2111         return int($1) if $value =~ m/^(\d+)$/;
2112         die "type check ('integer') failed - got '$value'\n";
2113     } elsif ($type eq 'number') {
2114         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2115         die "type check ('number') failed - got '$value'\n";
2116     } elsif ($type eq 'string') {
2117         if (my $fmt = $confdesc->{$key}->{format}) {
2118             PVE::JSONSchema::check_format($fmt, $value);
2119             return $value;
2120         }
2121         $value =~ s/^\"(.*)\"$/$1/;
2122         return $value;
2123     } else {
2124         die "internal error"
2125     }
2126 }
2127
2128 sub destroy_vm {
2129     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2130
2131     my $conf = PVE::QemuConfig->load_config($vmid);
2132
2133     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2134
2135     if ($conf->{template}) {
2136         # check if any base image is still used by a linked clone
2137         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2138                 my ($ds, $drive) = @_;
2139                 return if drive_is_cdrom($drive);
2140
2141                 my $volid = $drive->{file};
2142                 return if !$volid || $volid =~ m|^/|;
2143
2144                 die "base volume '$volid' is still in use by linked cloned\n"
2145                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2146
2147         });
2148     }
2149
2150     my $remove_owned_drive = sub {
2151         my ($ds, $drive) = @_;
2152         return if drive_is_cdrom($drive, 1);
2153
2154         my $volid = $drive->{file};
2155         return if !$volid || $volid =~ m|^/|;
2156
2157         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2158         return if !$path || !$owner || ($owner != $vmid);
2159
2160         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2161         warn "Could not remove disk '$volid', check manually: $@" if $@;
2162     };
2163
2164     # only remove disks owned by this VM (referenced in the config)
2165     my $include_opts = {
2166         include_unused => 1,
2167         extra_keys => ['vmstate'],
2168     };
2169     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2170
2171     for my $snap (values %{$conf->{snapshots}}) {
2172         next if !defined($snap->{vmstate});
2173         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2174         next if !defined($drive);
2175         $remove_owned_drive->('vmstate', $drive);
2176     }
2177
2178     if ($purge_unreferenced) { # also remove unreferenced disk
2179         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2180         PVE::Storage::foreach_volid($vmdisks, sub {
2181             my ($volid, $sid, $volname, $d) = @_;
2182             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2183             warn $@ if $@;
2184         });
2185     }
2186
2187     if (defined $replacement_conf) {
2188         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2189     } else {
2190         PVE::QemuConfig->destroy_config($vmid);
2191     }
2192 }
2193
2194 sub parse_vm_config {
2195     my ($filename, $raw) = @_;
2196
2197     return if !defined($raw);
2198
2199     my $res = {
2200         digest => Digest::SHA::sha1_hex($raw),
2201         snapshots => {},
2202         pending => {},
2203     };
2204
2205     $filename =~ m|/qemu-server/(\d+)\.conf$|
2206         || die "got strange filename '$filename'";
2207
2208     my $vmid = $1;
2209
2210     my $conf = $res;
2211     my $descr;
2212     my $section = '';
2213
2214     my @lines = split(/\n/, $raw);
2215     foreach my $line (@lines) {
2216         next if $line =~ m/^\s*$/;
2217
2218         if ($line =~ m/^\[PENDING\]\s*$/i) {
2219             $section = 'pending';
2220             if (defined($descr)) {
2221                 $descr =~ s/\s+$//;
2222                 $conf->{description} = $descr;
2223             }
2224             $descr = undef;
2225             $conf = $res->{$section} = {};
2226             next;
2227
2228         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2229             $section = $1;
2230             if (defined($descr)) {
2231                 $descr =~ s/\s+$//;
2232                 $conf->{description} = $descr;
2233             }
2234             $descr = undef;
2235             $conf = $res->{snapshots}->{$section} = {};
2236             next;
2237         }
2238
2239         if ($line =~ m/^\#(.*)\s*$/) {
2240             $descr = '' if !defined($descr);
2241             $descr .= PVE::Tools::decode_text($1) . "\n";
2242             next;
2243         }
2244
2245         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2246             $descr = '' if !defined($descr);
2247             $descr .= PVE::Tools::decode_text($2);
2248         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2249             $conf->{snapstate} = $1;
2250         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2251             my $key = $1;
2252             my $value = $2;
2253             $conf->{$key} = $value;
2254         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2255             my $value = $1;
2256             if ($section eq 'pending') {
2257                 $conf->{delete} = $value; # we parse this later
2258             } else {
2259                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2260             }
2261         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2262             my $key = $1;
2263             my $value = $2;
2264             eval { $value = check_type($key, $value); };
2265             if ($@) {
2266                 warn "vm $vmid - unable to parse value of '$key' - $@";
2267             } else {
2268                 $key = 'ide2' if $key eq 'cdrom';
2269                 my $fmt = $confdesc->{$key}->{format};
2270                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2271                     my $v = parse_drive($key, $value);
2272                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2273                         $v->{file} = $volid;
2274                         $value = print_drive($v);
2275                     } else {
2276                         warn "vm $vmid - unable to parse value of '$key'\n";
2277                         next;
2278                     }
2279                 }
2280
2281                 $conf->{$key} = $value;
2282             }
2283         } else {
2284             warn "vm $vmid - unable to parse config: $line\n";
2285         }
2286     }
2287
2288     if (defined($descr)) {
2289         $descr =~ s/\s+$//;
2290         $conf->{description} = $descr;
2291     }
2292     delete $res->{snapstate}; # just to be sure
2293
2294     return $res;
2295 }
2296
2297 sub write_vm_config {
2298     my ($filename, $conf) = @_;
2299
2300     delete $conf->{snapstate}; # just to be sure
2301
2302     if ($conf->{cdrom}) {
2303         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2304         $conf->{ide2} = $conf->{cdrom};
2305         delete $conf->{cdrom};
2306     }
2307
2308     # we do not use 'smp' any longer
2309     if ($conf->{sockets}) {
2310         delete $conf->{smp};
2311     } elsif ($conf->{smp}) {
2312         $conf->{sockets} = $conf->{smp};
2313         delete $conf->{cores};
2314         delete $conf->{smp};
2315     }
2316
2317     my $used_volids = {};
2318
2319     my $cleanup_config = sub {
2320         my ($cref, $pending, $snapname) = @_;
2321
2322         foreach my $key (keys %$cref) {
2323             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2324                 $key eq 'snapstate' || $key eq 'pending';
2325             my $value = $cref->{$key};
2326             if ($key eq 'delete') {
2327                 die "propertry 'delete' is only allowed in [PENDING]\n"
2328                     if !$pending;
2329                 # fixme: check syntax?
2330                 next;
2331             }
2332             eval { $value = check_type($key, $value); };
2333             die "unable to parse value of '$key' - $@" if $@;
2334
2335             $cref->{$key} = $value;
2336
2337             if (!$snapname && is_valid_drivename($key)) {
2338                 my $drive = parse_drive($key, $value);
2339                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2340             }
2341         }
2342     };
2343
2344     &$cleanup_config($conf);
2345
2346     &$cleanup_config($conf->{pending}, 1);
2347
2348     foreach my $snapname (keys %{$conf->{snapshots}}) {
2349         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2350         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2351     }
2352
2353     # remove 'unusedX' settings if we re-add a volume
2354     foreach my $key (keys %$conf) {
2355         my $value = $conf->{$key};
2356         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2357             delete $conf->{$key};
2358         }
2359     }
2360
2361     my $generate_raw_config = sub {
2362         my ($conf, $pending) = @_;
2363
2364         my $raw = '';
2365
2366         # add description as comment to top of file
2367         if (defined(my $descr = $conf->{description})) {
2368             if ($descr) {
2369                 foreach my $cl (split(/\n/, $descr)) {
2370                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2371                 }
2372             } else {
2373                 $raw .= "#\n" if $pending;
2374             }
2375         }
2376
2377         foreach my $key (sort keys %$conf) {
2378             next if $key =~ /^(digest|description|pending|snapshots)$/;
2379             $raw .= "$key: $conf->{$key}\n";
2380         }
2381         return $raw;
2382     };
2383
2384     my $raw = &$generate_raw_config($conf);
2385
2386     if (scalar(keys %{$conf->{pending}})){
2387         $raw .= "\n[PENDING]\n";
2388         $raw .= &$generate_raw_config($conf->{pending}, 1);
2389     }
2390
2391     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2392         $raw .= "\n[$snapname]\n";
2393         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2394     }
2395
2396     return $raw;
2397 }
2398
2399 sub load_defaults {
2400
2401     my $res = {};
2402
2403     # we use static defaults from our JSON schema configuration
2404     foreach my $key (keys %$confdesc) {
2405         if (defined(my $default = $confdesc->{$key}->{default})) {
2406             $res->{$key} = $default;
2407         }
2408     }
2409
2410     return $res;
2411 }
2412
2413 sub config_list {
2414     my $vmlist = PVE::Cluster::get_vmlist();
2415     my $res = {};
2416     return $res if !$vmlist || !$vmlist->{ids};
2417     my $ids = $vmlist->{ids};
2418     my $nodename = nodename();
2419
2420     foreach my $vmid (keys %$ids) {
2421         my $d = $ids->{$vmid};
2422         next if !$d->{node} || $d->{node} ne $nodename;
2423         next if !$d->{type} || $d->{type} ne 'qemu';
2424         $res->{$vmid}->{exists} = 1;
2425     }
2426     return $res;
2427 }
2428
2429 # test if VM uses local resources (to prevent migration)
2430 sub check_local_resources {
2431     my ($conf, $noerr) = @_;
2432
2433     my @loc_res = ();
2434
2435     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2436     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2437
2438     push @loc_res, "ivshmem" if $conf->{ivshmem};
2439
2440     foreach my $k (keys %$conf) {
2441         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2442         # sockets are safe: they will recreated be on the target side post-migrate
2443         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2444         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2445     }
2446
2447     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2448
2449     return \@loc_res;
2450 }
2451
2452 # check if used storages are available on all nodes (use by migrate)
2453 sub check_storage_availability {
2454     my ($storecfg, $conf, $node) = @_;
2455
2456     PVE::QemuConfig->foreach_volume($conf, sub {
2457         my ($ds, $drive) = @_;
2458
2459         my $volid = $drive->{file};
2460         return if !$volid;
2461
2462         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2463         return if !$sid;
2464
2465         # check if storage is available on both nodes
2466         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2467         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2468
2469         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2470
2471         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2472             if !$scfg->{content}->{$vtype};
2473    });
2474 }
2475
2476 # list nodes where all VM images are available (used by has_feature API)
2477 sub shared_nodes {
2478     my ($conf, $storecfg) = @_;
2479
2480     my $nodelist = PVE::Cluster::get_nodelist();
2481     my $nodehash = { map { $_ => 1 } @$nodelist };
2482     my $nodename = nodename();
2483
2484     PVE::QemuConfig->foreach_volume($conf, sub {
2485         my ($ds, $drive) = @_;
2486
2487         my $volid = $drive->{file};
2488         return if !$volid;
2489
2490         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2491         if ($storeid) {
2492             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2493             if ($scfg->{disable}) {
2494                 $nodehash = {};
2495             } elsif (my $avail = $scfg->{nodes}) {
2496                 foreach my $node (keys %$nodehash) {
2497                     delete $nodehash->{$node} if !$avail->{$node};
2498                 }
2499             } elsif (!$scfg->{shared}) {
2500                 foreach my $node (keys %$nodehash) {
2501                     delete $nodehash->{$node} if $node ne $nodename
2502                 }
2503             }
2504         }
2505     });
2506
2507     return $nodehash
2508 }
2509
2510 sub check_local_storage_availability {
2511     my ($conf, $storecfg) = @_;
2512
2513     my $nodelist = PVE::Cluster::get_nodelist();
2514     my $nodehash = { map { $_ => {} } @$nodelist };
2515
2516     PVE::QemuConfig->foreach_volume($conf, sub {
2517         my ($ds, $drive) = @_;
2518
2519         my $volid = $drive->{file};
2520         return if !$volid;
2521
2522         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2523         if ($storeid) {
2524             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2525
2526             if ($scfg->{disable}) {
2527                 foreach my $node (keys %$nodehash) {
2528                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2529                 }
2530             } elsif (my $avail = $scfg->{nodes}) {
2531                 foreach my $node (keys %$nodehash) {
2532                     if (!$avail->{$node}) {
2533                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2534                     }
2535                 }
2536             }
2537         }
2538     });
2539
2540     foreach my $node (values %$nodehash) {
2541         if (my $unavail = $node->{unavailable_storages}) {
2542             $node->{unavailable_storages} = [ sort keys %$unavail ];
2543         }
2544     }
2545
2546     return $nodehash
2547 }
2548
2549 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2550 sub check_running {
2551     my ($vmid, $nocheck, $node) = @_;
2552
2553     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2554     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2555 }
2556
2557 sub vzlist {
2558
2559     my $vzlist = config_list();
2560
2561     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2562
2563     while (defined(my $de = $fd->read)) {
2564         next if $de !~ m/^(\d+)\.pid$/;
2565         my $vmid = $1;
2566         next if !defined($vzlist->{$vmid});
2567         if (my $pid = check_running($vmid)) {
2568             $vzlist->{$vmid}->{pid} = $pid;
2569         }
2570     }
2571
2572     return $vzlist;
2573 }
2574
2575 our $vmstatus_return_properties = {
2576     vmid => get_standard_option('pve-vmid'),
2577     status => {
2578         description => "Qemu process status.",
2579         type => 'string',
2580         enum => ['stopped', 'running'],
2581     },
2582     maxmem => {
2583         description => "Maximum memory in bytes.",
2584         type => 'integer',
2585         optional => 1,
2586         renderer => 'bytes',
2587     },
2588     maxdisk => {
2589         description => "Root disk size in bytes.",
2590         type => 'integer',
2591         optional => 1,
2592         renderer => 'bytes',
2593     },
2594     name => {
2595         description => "VM name.",
2596         type => 'string',
2597         optional => 1,
2598     },
2599     qmpstatus => {
2600         description => "Qemu QMP agent status.",
2601         type => 'string',
2602         optional => 1,
2603     },
2604     pid => {
2605         description => "PID of running qemu process.",
2606         type => 'integer',
2607         optional => 1,
2608     },
2609     uptime => {
2610         description => "Uptime.",
2611         type => 'integer',
2612         optional => 1,
2613         renderer => 'duration',
2614     },
2615     cpus => {
2616         description => "Maximum usable CPUs.",
2617         type => 'number',
2618         optional => 1,
2619     },
2620     lock => {
2621         description => "The current config lock, if any.",
2622         type => 'string',
2623         optional => 1,
2624     },
2625     tags => {
2626         description  => "The current configured tags, if any",
2627         type => 'string',
2628         optional => 1,
2629     },
2630     'running-machine' => {
2631         description => "The currently running machine type (if running).",
2632         type => 'string',
2633         optional => 1,
2634     },
2635     'running-qemu' => {
2636         description => "The currently running QEMU version (if running).",
2637         type => 'string',
2638         optional => 1,
2639     },
2640 };
2641
2642 my $last_proc_pid_stat;
2643
2644 # get VM status information
2645 # This must be fast and should not block ($full == false)
2646 # We only query KVM using QMP if $full == true (this can be slow)
2647 sub vmstatus {
2648     my ($opt_vmid, $full) = @_;
2649
2650     my $res = {};
2651
2652     my $storecfg = PVE::Storage::config();
2653
2654     my $list = vzlist();
2655     my $defaults = load_defaults();
2656
2657     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2658
2659     my $cpucount = $cpuinfo->{cpus} || 1;
2660
2661     foreach my $vmid (keys %$list) {
2662         next if $opt_vmid && ($vmid ne $opt_vmid);
2663
2664         my $conf = PVE::QemuConfig->load_config($vmid);
2665
2666         my $d = { vmid => int($vmid) };
2667         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2668
2669         # fixme: better status?
2670         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2671
2672         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2673         if (defined($size)) {
2674             $d->{disk} = 0; # no info available
2675             $d->{maxdisk} = $size;
2676         } else {
2677             $d->{disk} = 0;
2678             $d->{maxdisk} = 0;
2679         }
2680
2681         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2682             * ($conf->{cores} || $defaults->{cores});
2683         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2684         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2685
2686         $d->{name} = $conf->{name} || "VM $vmid";
2687         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2688             : $defaults->{memory}*(1024*1024);
2689
2690         if ($conf->{balloon}) {
2691             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2692             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2693                 : $defaults->{shares};
2694         }
2695
2696         $d->{uptime} = 0;
2697         $d->{cpu} = 0;
2698         $d->{mem} = 0;
2699
2700         $d->{netout} = 0;
2701         $d->{netin} = 0;
2702
2703         $d->{diskread} = 0;
2704         $d->{diskwrite} = 0;
2705
2706         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2707
2708         $d->{serial} = 1 if conf_has_serial($conf);
2709         $d->{lock} = $conf->{lock} if $conf->{lock};
2710         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2711
2712         $res->{$vmid} = $d;
2713     }
2714
2715     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2716     foreach my $dev (keys %$netdev) {
2717         next if $dev !~ m/^tap([1-9]\d*)i/;
2718         my $vmid = $1;
2719         my $d = $res->{$vmid};
2720         next if !$d;
2721
2722         $d->{netout} += $netdev->{$dev}->{receive};
2723         $d->{netin} += $netdev->{$dev}->{transmit};
2724
2725         if ($full) {
2726             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2727             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2728         }
2729
2730     }
2731
2732     my $ctime = gettimeofday;
2733
2734     foreach my $vmid (keys %$list) {
2735
2736         my $d = $res->{$vmid};
2737         my $pid = $d->{pid};
2738         next if !$pid;
2739
2740         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2741         next if !$pstat; # not running
2742
2743         my $used = $pstat->{utime} + $pstat->{stime};
2744
2745         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2746
2747         if ($pstat->{vsize}) {
2748             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2749         }
2750
2751         my $old = $last_proc_pid_stat->{$pid};
2752         if (!$old) {
2753             $last_proc_pid_stat->{$pid} = {
2754                 time => $ctime,
2755                 used => $used,
2756                 cpu => 0,
2757             };
2758             next;
2759         }
2760
2761         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2762
2763         if ($dtime > 1000) {
2764             my $dutime = $used -  $old->{used};
2765
2766             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2767             $last_proc_pid_stat->{$pid} = {
2768                 time => $ctime,
2769                 used => $used,
2770                 cpu => $d->{cpu},
2771             };
2772         } else {
2773             $d->{cpu} = $old->{cpu};
2774         }
2775     }
2776
2777     return $res if !$full;
2778
2779     my $qmpclient = PVE::QMPClient->new();
2780
2781     my $ballooncb = sub {
2782         my ($vmid, $resp) = @_;
2783
2784         my $info = $resp->{'return'};
2785         return if !$info->{max_mem};
2786
2787         my $d = $res->{$vmid};
2788
2789         # use memory assigned to VM
2790         $d->{maxmem} = $info->{max_mem};
2791         $d->{balloon} = $info->{actual};
2792
2793         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2794             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2795             $d->{freemem} = $info->{free_mem};
2796         }
2797
2798         $d->{ballooninfo} = $info;
2799     };
2800
2801     my $blockstatscb = sub {
2802         my ($vmid, $resp) = @_;
2803         my $data = $resp->{'return'} || [];
2804         my $totalrdbytes = 0;
2805         my $totalwrbytes = 0;
2806
2807         for my $blockstat (@$data) {
2808             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2809             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2810
2811             $blockstat->{device} =~ s/drive-//;
2812             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2813         }
2814         $res->{$vmid}->{diskread} = $totalrdbytes;
2815         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2816     };
2817
2818     my $machinecb = sub {
2819         my ($vmid, $resp) = @_;
2820         my $data = $resp->{'return'} || [];
2821
2822         $res->{$vmid}->{'running-machine'} =
2823             PVE::QemuServer::Machine::current_from_query_machines($data);
2824     };
2825
2826     my $versioncb = sub {
2827         my ($vmid, $resp) = @_;
2828         my $data = $resp->{'return'} // {};
2829         my $version = 'unknown';
2830
2831         if (my $v = $data->{qemu}) {
2832             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2833         }
2834
2835         $res->{$vmid}->{'running-qemu'} = $version;
2836     };
2837
2838     my $statuscb = sub {
2839         my ($vmid, $resp) = @_;
2840
2841         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2842         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2843         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2844         # this fails if ballon driver is not loaded, so this must be
2845         # the last commnand (following command are aborted if this fails).
2846         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2847
2848         my $status = 'unknown';
2849         if (!defined($status = $resp->{'return'}->{status})) {
2850             warn "unable to get VM status\n";
2851             return;
2852         }
2853
2854         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2855     };
2856
2857     foreach my $vmid (keys %$list) {
2858         next if $opt_vmid && ($vmid ne $opt_vmid);
2859         next if !$res->{$vmid}->{pid}; # not running
2860         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2861     }
2862
2863     $qmpclient->queue_execute(undef, 2);
2864
2865     foreach my $vmid (keys %$list) {
2866         next if $opt_vmid && ($vmid ne $opt_vmid);
2867         next if !$res->{$vmid}->{pid}; #not running
2868
2869         # we can't use the $qmpclient since it might have already aborted on
2870         # 'query-balloon', but this might also fail for older versions...
2871         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2872         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2873     }
2874
2875     foreach my $vmid (keys %$list) {
2876         next if $opt_vmid && ($vmid ne $opt_vmid);
2877         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2878     }
2879
2880     return $res;
2881 }
2882
2883 sub conf_has_serial {
2884     my ($conf) = @_;
2885
2886     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2887         if ($conf->{"serial$i"}) {
2888             return 1;
2889         }
2890     }
2891
2892     return 0;
2893 }
2894
2895 sub conf_has_audio {
2896     my ($conf, $id) = @_;
2897
2898     $id //= 0;
2899     my $audio = $conf->{"audio$id"};
2900     return if !defined($audio);
2901
2902     my $audioproperties = parse_property_string($audio_fmt, $audio);
2903     my $audiodriver = $audioproperties->{driver} // 'spice';
2904
2905     return {
2906         dev => $audioproperties->{device},
2907         dev_id => "audiodev$id",
2908         backend => $audiodriver,
2909         backend_id => "$audiodriver-backend${id}",
2910     };
2911 }
2912
2913 sub audio_devs {
2914     my ($audio, $audiopciaddr, $machine_version) = @_;
2915
2916     my $devs = [];
2917
2918     my $id = $audio->{dev_id};
2919     my $audiodev = "";
2920     if (min_version($machine_version, 4, 2)) {
2921         $audiodev = ",audiodev=$audio->{backend_id}";
2922     }
2923
2924     if ($audio->{dev} eq 'AC97') {
2925         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2926     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2927         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2928         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2929         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2930     } else {
2931         die "unkown audio device '$audio->{dev}', implement me!";
2932     }
2933
2934     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2935
2936     return $devs;
2937 }
2938
2939 sub vga_conf_has_spice {
2940     my ($vga) = @_;
2941
2942     my $vgaconf = parse_vga($vga);
2943     my $vgatype = $vgaconf->{type};
2944     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2945
2946     return $1 || 1;
2947 }
2948
2949 sub is_native($) {
2950     my ($arch) = @_;
2951     return get_host_arch() eq $arch;
2952 }
2953
2954 sub get_vm_arch {
2955     my ($conf) = @_;
2956     return $conf->{arch} // get_host_arch();
2957 }
2958
2959 my $default_machines = {
2960     x86_64 => 'pc',
2961     aarch64 => 'virt',
2962 };
2963
2964 sub get_installed_machine_version {
2965     my ($kvmversion) = @_;
2966     $kvmversion = kvm_user_version() if !defined($kvmversion);
2967     $kvmversion =~ m/^(\d+\.\d+)/;
2968     return $1;
2969 }
2970
2971 sub windows_get_pinned_machine_version {
2972     my ($machine, $base_version, $kvmversion) = @_;
2973
2974     my $pin_version = $base_version;
2975     if (!defined($base_version) ||
2976         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2977     ) {
2978         $pin_version = get_installed_machine_version($kvmversion);
2979     }
2980     if (!$machine || $machine eq 'pc') {
2981         $machine = "pc-i440fx-$pin_version";
2982     } elsif ($machine eq 'q35') {
2983         $machine = "pc-q35-$pin_version";
2984     } elsif ($machine eq 'virt') {
2985         $machine = "virt-$pin_version";
2986     } else {
2987         warn "unknown machine type '$machine', not touching that!\n";
2988     }
2989
2990     return $machine;
2991 }
2992
2993 sub get_vm_machine {
2994     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2995
2996     my $machine = $forcemachine || $conf->{machine};
2997
2998     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2999         $kvmversion //= kvm_user_version();
3000         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3001         # layout which confuses windows quite a bit and may result in various regressions..
3002         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3003         if (windows_version($conf->{ostype})) {
3004             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3005         }
3006         $arch //= 'x86_64';
3007         $machine ||= $default_machines->{$arch};
3008         if ($add_pve_version) {
3009             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3010             $machine .= "+pve$pvever";
3011         }
3012     }
3013
3014     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3015         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3016         $machine = $1 if $is_pxe;
3017
3018         # for version-pinned machines that do not include a pve-version (e.g.
3019         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3020         $machine .= '+pve0';
3021
3022         $machine .= '.pxe' if $is_pxe;
3023     }
3024
3025     return $machine;
3026 }
3027
3028 sub get_ovmf_files($) {
3029     my ($arch) = @_;
3030
3031     my $ovmf = $OVMF->{$arch}
3032         or die "no OVMF images known for architecture '$arch'\n";
3033
3034     return @$ovmf;
3035 }
3036
3037 my $Arch2Qemu = {
3038     aarch64 => '/usr/bin/qemu-system-aarch64',
3039     x86_64 => '/usr/bin/qemu-system-x86_64',
3040 };
3041 sub get_command_for_arch($) {
3042     my ($arch) = @_;
3043     return '/usr/bin/kvm' if is_native($arch);
3044
3045     my $cmd = $Arch2Qemu->{$arch}
3046         or die "don't know how to emulate architecture '$arch'\n";
3047     return $cmd;
3048 }
3049
3050 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3051 # to use in a QEMU command line (-cpu element), first array_intersect the result
3052 # of query_supported_ with query_understood_. This is necessary because:
3053 #
3054 # a) query_understood_ returns flags the host cannot use and
3055 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3056 #    flags, but CPU settings - with most of them being flags. Those settings
3057 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3058 #
3059 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3060 # expensive. If you need the value returned from this, you can get it much
3061 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3062 # $accel being 'kvm' or 'tcg'.
3063 #
3064 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3065 # changes, automatically populating pmxcfs.
3066 #
3067 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3068 # since kvm and tcg machines support different flags
3069 #
3070 sub query_supported_cpu_flags {
3071     my ($arch) = @_;
3072
3073     $arch //= get_host_arch();
3074     my $default_machine = $default_machines->{$arch};
3075
3076     my $flags = {};
3077
3078     # FIXME: Once this is merged, the code below should work for ARM as well:
3079     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3080     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3081         $arch eq "aarch64";
3082
3083     my $kvm_supported = defined(kvm_version());
3084     my $qemu_cmd = get_command_for_arch($arch);
3085     my $fakevmid = -1;
3086     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3087
3088     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3089     my $query_supported_run_qemu = sub {
3090         my ($kvm) = @_;
3091
3092         my $flags = {};
3093         my $cmd = [
3094             $qemu_cmd,
3095             '-machine', $default_machine,
3096             '-display', 'none',
3097             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3098             '-mon', 'chardev=qmp,mode=control',
3099             '-pidfile', $pidfile,
3100             '-S', '-daemonize'
3101         ];
3102
3103         if (!$kvm) {
3104             push @$cmd, '-accel', 'tcg';
3105         }
3106
3107         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3108         die "QEMU flag querying VM exited with code " . $rc if $rc;
3109
3110         eval {
3111             my $cmd_result = mon_cmd(
3112                 $fakevmid,
3113                 'query-cpu-model-expansion',
3114                 type => 'full',
3115                 model => { name => 'host' }
3116             );
3117
3118             my $props = $cmd_result->{model}->{props};
3119             foreach my $prop (keys %$props) {
3120                 next if $props->{$prop} ne '1';
3121                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3122                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3123                 # We only keep those with underscores, to match /proc/cpuinfo
3124                 $prop =~ s/\.|-/_/g;
3125                 $flags->{$prop} = 1;
3126             }
3127         };
3128         my $err = $@;
3129
3130         # force stop with 10 sec timeout and 'nocheck'
3131         # always stop, even if QMP failed
3132         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3133
3134         die $err if $err;
3135
3136         return [ sort keys %$flags ];
3137     };
3138
3139     # We need to query QEMU twice, since KVM and TCG have different supported flags
3140     PVE::QemuConfig->lock_config($fakevmid, sub {
3141         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3142         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3143
3144         if ($kvm_supported) {
3145             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3146             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3147         }
3148     });
3149
3150     return $flags;
3151 }
3152
3153 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3154 my $understood_cpu_flag_dir = "/usr/share/kvm";
3155 sub query_understood_cpu_flags {
3156     my $arch = get_host_arch();
3157     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3158
3159     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3160         if ! -e $filepath;
3161
3162     my $raw = file_get_contents($filepath);
3163     $raw =~ s/^\s+|\s+$//g;
3164     my @flags = split(/\s+/, $raw);
3165
3166     return \@flags;
3167 }
3168
3169 sub config_to_command {
3170     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3171         $pbs_backing) = @_;
3172
3173     my $cmd = [];
3174     my $globalFlags = [];
3175     my $machineFlags = [];
3176     my $rtcFlags = [];
3177     my $devices = [];
3178     my $pciaddr = '';
3179     my $bridges = {};
3180     my $ostype = $conf->{ostype};
3181     my $winversion = windows_version($ostype);
3182     my $kvm = $conf->{kvm};
3183     my $nodename = nodename();
3184
3185     my $arch = get_vm_arch($conf);
3186     my $kvm_binary = get_command_for_arch($arch);
3187     my $kvmver = kvm_user_version($kvm_binary);
3188
3189     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3190         $kvmver //= "undefined";
3191         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3192     }
3193
3194     my $add_pve_version = min_version($kvmver, 4, 1);
3195
3196     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3197     my $machine_version = extract_version($machine_type, $kvmver);
3198     $kvm //= 1 if is_native($arch);
3199
3200     $machine_version =~ m/(\d+)\.(\d+)/;
3201     my ($machine_major, $machine_minor) = ($1, $2);
3202
3203     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3204         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3205     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3206         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3207             ." please upgrade node '$nodename'\n"
3208     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3209         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3210         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3211             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3212             ." node '$nodename'\n";
3213     }
3214
3215     # if a specific +pve version is required for a feature, use $version_guard
3216     # instead of min_version to allow machines to be run with the minimum
3217     # required version
3218     my $required_pve_version = 0;
3219     my $version_guard = sub {
3220         my ($major, $minor, $pve) = @_;
3221         return 0 if !min_version($machine_version, $major, $minor, $pve);
3222         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3223         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3224         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3225         return 1;
3226     };
3227
3228     if ($kvm && !defined kvm_version()) {
3229         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3230             ." or enable in BIOS.\n";
3231     }
3232
3233     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3234     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3235     my $use_old_bios_files = undef;
3236     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3237
3238     my $cpuunits = defined($conf->{cpuunits}) ?
3239             $conf->{cpuunits} : $defaults->{cpuunits};
3240
3241     push @$cmd, $kvm_binary;
3242
3243     push @$cmd, '-id', $vmid;
3244
3245     my $vmname = $conf->{name} || "vm$vmid";
3246
3247     push @$cmd, '-name', $vmname;
3248
3249     push @$cmd, '-no-shutdown';
3250
3251     my $use_virtio = 0;
3252
3253     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3254     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3255     push @$cmd, '-mon', "chardev=qmp,mode=control";
3256
3257     if (min_version($machine_version, 2, 12)) {
3258         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3259         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3260     }
3261
3262     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3263
3264     push @$cmd, '-daemonize';
3265
3266     if ($conf->{smbios1}) {
3267         my $smbios_conf = parse_smbios1($conf->{smbios1});
3268         if ($smbios_conf->{base64}) {
3269             # Do not pass base64 flag to qemu
3270             delete $smbios_conf->{base64};
3271             my $smbios_string = "";
3272             foreach my $key (keys %$smbios_conf) {
3273                 my $value;
3274                 if ($key eq "uuid") {
3275                     $value = $smbios_conf->{uuid}
3276                 } else {
3277                     $value = decode_base64($smbios_conf->{$key});
3278                 }
3279                 # qemu accepts any binary data, only commas need escaping by double comma
3280                 $value =~ s/,/,,/g;
3281                 $smbios_string .= "," . $key . "=" . $value if $value;
3282             }
3283             push @$cmd, '-smbios', "type=1" . $smbios_string;
3284         } else {
3285             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3286         }
3287     }
3288
3289     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3290         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3291         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3292
3293         my ($path, $format);
3294         my $read_only_str = '';
3295         if (my $efidisk = $conf->{efidisk0}) {
3296             my $d = parse_drive('efidisk0', $efidisk);
3297             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3298             $format = $d->{format};
3299             if ($storeid) {
3300                 $path = PVE::Storage::path($storecfg, $d->{file});
3301                 if (!defined($format)) {
3302                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3303                     $format = qemu_img_format($scfg, $volname);
3304                 }
3305             } else {
3306                 $path = $d->{file};
3307                 die "efidisk format must be specified\n"
3308                     if !defined($format);
3309             }
3310
3311             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3312         } else {
3313             warn "no efidisk configured! Using temporary efivars disk.\n";
3314             $path = "/tmp/$vmid-ovmf.fd";
3315             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3316             $format = 'raw';
3317         }
3318
3319         my $size_str = "";
3320
3321         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3322             $size_str = ",size=" . (-s $ovmf_vars);
3323         }
3324
3325         # on slower ceph clusters, booting without cache on efidisk can take a while, see #3329
3326         my $cache = $path =~ m/^rbd:/ ? ',cache=writeback' : '';
3327
3328         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3329         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3330     }
3331
3332     # load q35 config
3333     if ($q35) {
3334         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3335         if (min_version($machine_version, 4, 0)) {
3336             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3337         } else {
3338             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3339         }
3340     }
3341
3342     if ($conf->{vmgenid}) {
3343         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3344     }
3345
3346     # add usb controllers
3347     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3348         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3349     push @$devices, @usbcontrollers if @usbcontrollers;
3350     my $vga = parse_vga($conf->{vga});
3351
3352     my $qxlnum = vga_conf_has_spice($conf->{vga});
3353     $vga->{type} = 'qxl' if $qxlnum;
3354
3355     if (!$vga->{type}) {
3356         if ($arch eq 'aarch64') {
3357             $vga->{type} = 'virtio';
3358         } elsif (min_version($machine_version, 2, 9)) {
3359             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3360         } else {
3361             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3362         }
3363     }
3364
3365     # enable absolute mouse coordinates (needed by vnc)
3366     my $tablet;
3367     if (defined($conf->{tablet})) {
3368         $tablet = $conf->{tablet};
3369     } else {
3370         $tablet = $defaults->{tablet};
3371         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3372         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3373     }
3374
3375     if ($tablet) {
3376         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3377         my $kbd = print_keyboarddevice_full($conf, $arch);
3378         push @$devices, '-device', $kbd if defined($kbd);
3379     }
3380
3381     my $bootorder = device_bootorder($conf);
3382
3383     # host pci device passthrough
3384     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3385         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3386
3387     # usb devices
3388     my $usb_dev_features = {};
3389     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3390
3391     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3392         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3393     push @$devices, @usbdevices if @usbdevices;
3394
3395     # serial devices
3396     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3397         if (my $path = $conf->{"serial$i"}) {
3398             if ($path eq 'socket') {
3399                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3400                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3401                 # On aarch64, serial0 is the UART device. Qemu only allows
3402                 # connecting UART devices via the '-serial' command line, as
3403                 # the device has a fixed slot on the hardware...
3404                 if ($arch eq 'aarch64' && $i == 0) {
3405                     push @$devices, '-serial', "chardev:serial$i";
3406                 } else {
3407                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3408                 }
3409             } else {
3410                 die "no such serial device\n" if ! -c $path;
3411                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3412                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3413             }
3414         }
3415     }
3416
3417     # parallel devices
3418     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3419         if (my $path = $conf->{"parallel$i"}) {
3420             die "no such parallel device\n" if ! -c $path;
3421             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3422             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3423             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3424         }
3425     }
3426
3427     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3428         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3429         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3430         push @$devices, @$audio_devs;
3431     }
3432
3433     my $sockets = 1;
3434     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3435     $sockets = $conf->{sockets} if  $conf->{sockets};
3436
3437     my $cores = $conf->{cores} || 1;
3438
3439     my $maxcpus = $sockets * $cores;
3440
3441     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3442
3443     my $allowed_vcpus = $cpuinfo->{cpus};
3444
3445     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3446         if ($allowed_vcpus < $maxcpus);
3447
3448     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3449
3450         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3451         for (my $i = 2; $i <= $vcpus; $i++)  {
3452             my $cpustr = print_cpu_device($conf,$i);
3453             push @$cmd, '-device', $cpustr;
3454         }
3455
3456     } else {
3457
3458         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3459     }
3460     push @$cmd, '-nodefaults';
3461
3462     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3463
3464     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3465
3466     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3467
3468     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3469         push @$devices, '-device', print_vga_device(
3470             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3471         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3472         push @$cmd,  '-vnc', "unix:$socket,password=on";
3473     } else {
3474         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3475         push @$cmd, '-nographic';
3476     }
3477
3478     # time drift fix
3479     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3480     my $useLocaltime = $conf->{localtime};
3481
3482     if ($winversion >= 5) { # windows
3483         $useLocaltime = 1 if !defined($conf->{localtime});
3484
3485         # use time drift fix when acpi is enabled
3486         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3487             $tdf = 1 if !defined($conf->{tdf});
3488         }
3489     }
3490
3491     if ($winversion >= 6) {
3492         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3493         push @$cmd, '-no-hpet';
3494     }
3495
3496     push @$rtcFlags, 'driftfix=slew' if $tdf;
3497
3498     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3499         push @$rtcFlags, "base=$conf->{startdate}";
3500     } elsif ($useLocaltime) {
3501         push @$rtcFlags, 'base=localtime';
3502     }
3503
3504     if ($forcecpu) {
3505         push @$cmd, '-cpu', $forcecpu;
3506     } else {
3507         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3508     }
3509
3510     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3511
3512     push @$cmd, '-S' if $conf->{freeze};
3513
3514     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3515
3516     my $guest_agent = parse_guest_agent($conf);
3517
3518     if ($guest_agent->{enabled}) {
3519         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3520         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3521
3522         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3523             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3524             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3525             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3526         } elsif ($guest_agent->{type} eq 'isa') {
3527             push @$devices, '-device', "isa-serial,chardev=qga0";
3528         }
3529     }
3530
3531     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3532     if ($rng && $version_guard->(4, 1, 2)) {
3533         check_rng_source($rng->{source});
3534
3535         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3536         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3537         my $limiter_str = "";
3538         if ($max_bytes) {
3539             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3540         }
3541
3542         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3543         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3544         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3545     }
3546
3547     my $spice_port;
3548
3549     if ($qxlnum) {
3550         if ($qxlnum > 1) {
3551             if ($winversion){
3552                 for (my $i = 1; $i < $qxlnum; $i++){
3553                     push @$devices, '-device', print_vga_device(
3554                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3555                 }
3556             } else {
3557                 # assume other OS works like Linux
3558                 my ($ram, $vram) = ("134217728", "67108864");
3559                 if ($vga->{memory}) {
3560                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3561                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3562                 }
3563                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3564                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3565             }
3566         }
3567
3568         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3569
3570         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3571         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3572         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3573
3574         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3575         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3576         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3577
3578         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3579         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3580
3581         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3582         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3583         if ($spice_enhancement->{foldersharing}) {
3584             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3585             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3586         }
3587
3588         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3589         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3590             if $spice_enhancement->{videostreaming};
3591
3592         push @$devices, '-spice', "$spice_opts";
3593     }
3594
3595     # enable balloon by default, unless explicitly disabled
3596     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3597         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3598         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3599     }
3600
3601     if ($conf->{watchdog}) {
3602         my $wdopts = parse_watchdog($conf->{watchdog});
3603         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3604         my $watchdog = $wdopts->{model} || 'i6300esb';
3605         push @$devices, '-device', "$watchdog$pciaddr";
3606         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3607     }
3608
3609     my $vollist = [];
3610     my $scsicontroller = {};
3611     my $ahcicontroller = {};
3612     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3613
3614     # Add iscsi initiator name if available
3615     if (my $initiator = get_initiator_name()) {
3616         push @$devices, '-iscsi', "initiator-name=$initiator";
3617     }
3618
3619     PVE::QemuConfig->foreach_volume($conf, sub {
3620         my ($ds, $drive) = @_;
3621
3622         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3623             check_volume_storage_type($storecfg, $drive->{file});
3624             push @$vollist, $drive->{file};
3625         }
3626
3627         # ignore efidisk here, already added in bios/fw handling code above
3628         return if $drive->{interface} eq 'efidisk';
3629
3630         $use_virtio = 1 if $ds =~ m/^virtio/;
3631
3632         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3633
3634         if ($drive->{interface} eq 'virtio'){
3635            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3636         }
3637
3638         if ($drive->{interface} eq 'scsi') {
3639
3640             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3641
3642             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3643                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3644
3645             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3646             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3647
3648             my $iothread = '';
3649             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3650                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3651                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3652             } elsif ($drive->{iothread}) {
3653                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3654             }
3655
3656             my $queues = '';
3657             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3658                 $queues = ",num_queues=$drive->{queues}";
3659             }
3660
3661             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3662                 if !$scsicontroller->{$controller};
3663             $scsicontroller->{$controller}=1;
3664         }
3665
3666         if ($drive->{interface} eq 'sata') {
3667             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3668             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3669             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3670                 if !$ahcicontroller->{$controller};
3671             $ahcicontroller->{$controller}=1;
3672         }
3673
3674         my $pbs_conf = $pbs_backing->{$ds};
3675         my $pbs_name = undef;
3676         if ($pbs_conf) {
3677             $pbs_name = "drive-$ds-pbs";
3678             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3679         }
3680
3681         my $drive_cmd = print_drive_commandline_full(
3682             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3683
3684         # extra protection for templates, but SATA and IDE don't support it..
3685         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3686
3687         push @$devices, '-drive',$drive_cmd;
3688         push @$devices, '-device', print_drivedevice_full(
3689             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3690     });
3691
3692     for (my $i = 0; $i < $MAX_NETS; $i++) {
3693         my $netname = "net$i";
3694
3695         next if !$conf->{$netname};
3696         my $d = parse_net($conf->{$netname});
3697         next if !$d;
3698
3699         $use_virtio = 1 if $d->{model} eq 'virtio';
3700
3701         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3702
3703         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3704         push @$devices, '-netdev', $netdevfull;
3705
3706         my $netdevicefull = print_netdevice_full(
3707             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3708
3709         push @$devices, '-device', $netdevicefull;
3710     }
3711
3712     if ($conf->{ivshmem}) {
3713         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3714
3715         my $bus;
3716         if ($q35) {
3717             $bus = print_pcie_addr("ivshmem");
3718         } else {
3719             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3720         }
3721
3722         my $ivshmem_name = $ivshmem->{name} // $vmid;
3723         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3724
3725         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3726         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3727             .",size=$ivshmem->{size}M";
3728     }
3729
3730     # pci.4 is nested in pci.1
3731     $bridges->{1} = 1 if $bridges->{4};
3732
3733     if (!$q35) {
3734         # add pci bridges
3735         if (min_version($machine_version, 2, 3)) {
3736            $bridges->{1} = 1;
3737            $bridges->{2} = 1;
3738         }
3739
3740         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3741
3742     }
3743
3744     for my $k (sort {$b cmp $a} keys %$bridges) {
3745         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3746
3747         my $k_name = $k;
3748         if ($k == 2 && $legacy_igd) {
3749             $k_name = "$k-igd";
3750         }
3751         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3752
3753         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3754         if ($q35) {
3755             # add after -readconfig pve-q35.cfg
3756             splice @$devices, 2, 0, '-device', $devstr;
3757         } else {
3758             unshift @$devices, '-device', $devstr if $k > 0;
3759         }
3760     }
3761
3762     if (!$kvm) {
3763         push @$machineFlags, 'accel=tcg';
3764     }
3765
3766     my $machine_type_min = $machine_type;
3767     if ($add_pve_version) {
3768         $machine_type_min =~ s/\+pve\d+$//;
3769         $machine_type_min .= "+pve$required_pve_version";
3770     }
3771     push @$machineFlags, "type=${machine_type_min}";
3772
3773     push @$cmd, @$devices;
3774     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3775     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3776     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3777
3778     if (my $vmstate = $conf->{vmstate}) {
3779         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3780         push @$vollist, $vmstate;
3781         push @$cmd, '-loadstate', $statepath;
3782         print "activating and using '$vmstate' as vmstate\n";
3783     }
3784
3785     if (PVE::QemuConfig->is_template($conf)) {
3786         # needed to workaround base volumes being read-only
3787         push @$cmd, '-snapshot';
3788     }
3789
3790     # add custom args
3791     if ($conf->{args}) {
3792         my $aa = PVE::Tools::split_args($conf->{args});
3793         push @$cmd, @$aa;
3794     }
3795
3796     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3797 }
3798
3799 sub check_rng_source {
3800     my ($source) = @_;
3801
3802     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3803     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3804         if ! -e $source;
3805
3806     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3807     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3808         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3809         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3810         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3811             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3812             ." to the host.\n";
3813     }
3814 }
3815
3816 sub spice_port {
3817     my ($vmid) = @_;
3818
3819     my $res = mon_cmd($vmid, 'query-spice');
3820
3821     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3822 }
3823
3824 sub vm_devices_list {
3825     my ($vmid) = @_;
3826
3827     my $res = mon_cmd($vmid, 'query-pci');
3828     my $devices_to_check = [];
3829     my $devices = {};
3830     foreach my $pcibus (@$res) {
3831         push @$devices_to_check, @{$pcibus->{devices}},
3832     }
3833
3834     while (@$devices_to_check) {
3835         my $to_check = [];
3836         for my $d (@$devices_to_check) {
3837             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3838             next if !$d->{'pci_bridge'};
3839
3840             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3841             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3842         }
3843         $devices_to_check = $to_check;
3844     }
3845
3846     my $resblock = mon_cmd($vmid, 'query-block');
3847     foreach my $block (@$resblock) {
3848         if($block->{device} =~ m/^drive-(\S+)/){
3849                 $devices->{$1} = 1;
3850         }
3851     }
3852
3853     my $resmice = mon_cmd($vmid, 'query-mice');
3854     foreach my $mice (@$resmice) {
3855         if ($mice->{name} eq 'QEMU HID Tablet') {
3856             $devices->{tablet} = 1;
3857             last;
3858         }
3859     }
3860
3861     # for usb devices there is no query-usb
3862     # but we can iterate over the entries in
3863     # qom-list path=/machine/peripheral
3864     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3865     foreach my $per (@$resperipheral) {
3866         if ($per->{name} =~ m/^usb\d+$/) {
3867             $devices->{$per->{name}} = 1;
3868         }
3869     }
3870
3871     return $devices;
3872 }
3873
3874 sub vm_deviceplug {
3875     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3876
3877     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3878
3879     my $devices_list = vm_devices_list($vmid);
3880     return 1 if defined($devices_list->{$deviceid});
3881
3882     # add PCI bridge if we need it for the device
3883     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3884
3885     if ($deviceid eq 'tablet') {
3886
3887         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3888
3889     } elsif ($deviceid eq 'keyboard') {
3890
3891         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3892
3893     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3894
3895         die "usb hotplug currently not reliable\n";
3896         # since we can't reliably hot unplug all added usb devices and usb
3897         # passthrough breaks live migration we disable usb hotplugging for now
3898         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3899
3900     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3901
3902         qemu_iothread_add($vmid, $deviceid, $device);
3903
3904         qemu_driveadd($storecfg, $vmid, $device);
3905         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3906
3907         qemu_deviceadd($vmid, $devicefull);
3908         eval { qemu_deviceaddverify($vmid, $deviceid); };
3909         if (my $err = $@) {
3910             eval { qemu_drivedel($vmid, $deviceid); };
3911             warn $@ if $@;
3912             die $err;
3913         }
3914
3915     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3916
3917
3918         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3919         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3920         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3921
3922         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3923
3924         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3925             qemu_iothread_add($vmid, $deviceid, $device);
3926             $devicefull .= ",iothread=iothread-$deviceid";
3927         }
3928
3929         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3930             $devicefull .= ",num_queues=$device->{queues}";
3931         }
3932
3933         qemu_deviceadd($vmid, $devicefull);
3934         qemu_deviceaddverify($vmid, $deviceid);
3935
3936     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3937
3938         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3939         qemu_driveadd($storecfg, $vmid, $device);
3940
3941         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3942         eval { qemu_deviceadd($vmid, $devicefull); };
3943         if (my $err = $@) {
3944             eval { qemu_drivedel($vmid, $deviceid); };
3945             warn $@ if $@;
3946             die $err;
3947         }
3948
3949     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3950
3951         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3952
3953         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3954         my $use_old_bios_files = undef;
3955         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3956
3957         my $netdevicefull = print_netdevice_full(
3958             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3959         qemu_deviceadd($vmid, $netdevicefull);
3960         eval {
3961             qemu_deviceaddverify($vmid, $deviceid);
3962             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3963         };
3964         if (my $err = $@) {
3965             eval { qemu_netdevdel($vmid, $deviceid); };
3966             warn $@ if $@;
3967             die $err;
3968         }
3969
3970     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3971
3972         my $bridgeid = $2;
3973         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3974         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3975
3976         qemu_deviceadd($vmid, $devicefull);
3977         qemu_deviceaddverify($vmid, $deviceid);
3978
3979     } else {
3980         die "can't hotplug device '$deviceid'\n";
3981     }
3982
3983     return 1;
3984 }
3985
3986 # fixme: this should raise exceptions on error!
3987 sub vm_deviceunplug {
3988     my ($vmid, $conf, $deviceid) = @_;
3989
3990     my $devices_list = vm_devices_list($vmid);
3991     return 1 if !defined($devices_list->{$deviceid});
3992
3993     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3994     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3995
3996     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3997
3998         qemu_devicedel($vmid, $deviceid);
3999
4000     } elsif ($deviceid =~ m/^usb\d+$/) {
4001
4002         die "usb hotplug currently not reliable\n";
4003         # when unplugging usb devices this way, there may be remaining usb
4004         # controllers/hubs so we disable it for now
4005         #qemu_devicedel($vmid, $deviceid);
4006         #qemu_devicedelverify($vmid, $deviceid);
4007
4008     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4009
4010         qemu_devicedel($vmid, $deviceid);
4011         qemu_devicedelverify($vmid, $deviceid);
4012         qemu_drivedel($vmid, $deviceid);
4013         qemu_iothread_del($conf, $vmid, $deviceid);
4014
4015     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4016
4017         qemu_devicedel($vmid, $deviceid);
4018         qemu_devicedelverify($vmid, $deviceid);
4019         qemu_iothread_del($conf, $vmid, $deviceid);
4020
4021     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4022
4023         qemu_devicedel($vmid, $deviceid);
4024         qemu_drivedel($vmid, $deviceid);
4025         qemu_deletescsihw($conf, $vmid, $deviceid);
4026
4027     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4028
4029         qemu_devicedel($vmid, $deviceid);
4030         qemu_devicedelverify($vmid, $deviceid);
4031         qemu_netdevdel($vmid, $deviceid);
4032
4033     } else {
4034         die "can't unplug device '$deviceid'\n";
4035     }
4036
4037     return 1;
4038 }
4039
4040 sub qemu_deviceadd {
4041     my ($vmid, $devicefull) = @_;
4042
4043     $devicefull = "driver=".$devicefull;
4044     my %options =  split(/[=,]/, $devicefull);
4045
4046     mon_cmd($vmid, "device_add" , %options);
4047 }
4048
4049 sub qemu_devicedel {
4050     my ($vmid, $deviceid) = @_;
4051
4052     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4053 }
4054
4055 sub qemu_iothread_add {
4056     my($vmid, $deviceid, $device) = @_;
4057
4058     if ($device->{iothread}) {
4059         my $iothreads = vm_iothreads_list($vmid);
4060         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4061     }
4062 }
4063
4064 sub qemu_iothread_del {
4065     my($conf, $vmid, $deviceid) = @_;
4066
4067     my $confid = $deviceid;
4068     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4069         $confid = 'scsi' . $1;
4070     }
4071     my $device = parse_drive($confid, $conf->{$confid});
4072     if ($device->{iothread}) {
4073         my $iothreads = vm_iothreads_list($vmid);
4074         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4075     }
4076 }
4077
4078 sub qemu_objectadd {
4079     my($vmid, $objectid, $qomtype) = @_;
4080
4081     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4082
4083     return 1;
4084 }
4085
4086 sub qemu_objectdel {
4087     my($vmid, $objectid) = @_;
4088
4089     mon_cmd($vmid, "object-del", id => $objectid);
4090
4091     return 1;
4092 }
4093
4094 sub qemu_driveadd {
4095     my ($storecfg, $vmid, $device) = @_;
4096
4097     my $kvmver = get_running_qemu_version($vmid);
4098     my $io_uring = min_version($kvmver, 6, 0);
4099     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4100     $drive =~ s/\\/\\\\/g;
4101     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4102
4103     # If the command succeeds qemu prints: "OK"
4104     return 1 if $ret =~ m/OK/s;
4105
4106     die "adding drive failed: $ret\n";
4107 }
4108
4109 sub qemu_drivedel {
4110     my($vmid, $deviceid) = @_;
4111
4112     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4113     $ret =~ s/^\s+//;
4114
4115     return 1 if $ret eq "";
4116
4117     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4118     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4119
4120     die "deleting drive $deviceid failed : $ret\n";
4121 }
4122
4123 sub qemu_deviceaddverify {
4124     my ($vmid, $deviceid) = @_;
4125
4126     for (my $i = 0; $i <= 5; $i++) {
4127          my $devices_list = vm_devices_list($vmid);
4128          return 1 if defined($devices_list->{$deviceid});
4129          sleep 1;
4130     }
4131
4132     die "error on hotplug device '$deviceid'\n";
4133 }
4134
4135
4136 sub qemu_devicedelverify {
4137     my ($vmid, $deviceid) = @_;
4138
4139     # need to verify that the device is correctly removed as device_del
4140     # is async and empty return is not reliable
4141
4142     for (my $i = 0; $i <= 5; $i++) {
4143          my $devices_list = vm_devices_list($vmid);
4144          return 1 if !defined($devices_list->{$deviceid});
4145          sleep 1;
4146     }
4147
4148     die "error on hot-unplugging device '$deviceid'\n";
4149 }
4150
4151 sub qemu_findorcreatescsihw {
4152     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4153
4154     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4155
4156     my $scsihwid="$controller_prefix$controller";
4157     my $devices_list = vm_devices_list($vmid);
4158
4159     if(!defined($devices_list->{$scsihwid})) {
4160         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4161     }
4162
4163     return 1;
4164 }
4165
4166 sub qemu_deletescsihw {
4167     my ($conf, $vmid, $opt) = @_;
4168
4169     my $device = parse_drive($opt, $conf->{$opt});
4170
4171     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4172         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4173         return 1;
4174     }
4175
4176     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4177
4178     my $devices_list = vm_devices_list($vmid);
4179     foreach my $opt (keys %{$devices_list}) {
4180         if (is_valid_drivename($opt)) {
4181             my $drive = parse_drive($opt, $conf->{$opt});
4182             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4183                 return 1;
4184             }
4185         }
4186     }
4187
4188     my $scsihwid="scsihw$controller";
4189
4190     vm_deviceunplug($vmid, $conf, $scsihwid);
4191
4192     return 1;
4193 }
4194
4195 sub qemu_add_pci_bridge {
4196     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4197
4198     my $bridges = {};
4199
4200     my $bridgeid;
4201
4202     print_pci_addr($device, $bridges, $arch, $machine_type);
4203
4204     while (my ($k, $v) = each %$bridges) {
4205         $bridgeid = $k;
4206     }
4207     return 1 if !defined($bridgeid) || $bridgeid < 1;
4208
4209     my $bridge = "pci.$bridgeid";
4210     my $devices_list = vm_devices_list($vmid);
4211
4212     if (!defined($devices_list->{$bridge})) {
4213         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4214     }
4215
4216     return 1;
4217 }
4218
4219 sub qemu_set_link_status {
4220     my ($vmid, $device, $up) = @_;
4221
4222     mon_cmd($vmid, "set_link", name => $device,
4223                up => $up ? JSON::true : JSON::false);
4224 }
4225
4226 sub qemu_netdevadd {
4227     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4228
4229     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4230     my %options =  split(/[=,]/, $netdev);
4231
4232     if (defined(my $vhost = $options{vhost})) {
4233         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4234     }
4235
4236     if (defined(my $queues = $options{queues})) {
4237         $options{queues} = $queues + 0;
4238     }
4239
4240     mon_cmd($vmid, "netdev_add",  %options);
4241     return 1;
4242 }
4243
4244 sub qemu_netdevdel {
4245     my ($vmid, $deviceid) = @_;
4246
4247     mon_cmd($vmid, "netdev_del", id => $deviceid);
4248 }
4249
4250 sub qemu_usb_hotplug {
4251     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4252
4253     return if !$device;
4254
4255     # remove the old one first
4256     vm_deviceunplug($vmid, $conf, $deviceid);
4257
4258     # check if xhci controller is necessary and available
4259     if ($device->{usb3}) {
4260
4261         my $devicelist = vm_devices_list($vmid);
4262
4263         if (!$devicelist->{xhci}) {
4264             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4265             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4266         }
4267     }
4268     my $d = parse_usb_device($device->{host});
4269     $d->{usb3} = $device->{usb3};
4270
4271     # add the new one
4272     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4273 }
4274
4275 sub qemu_cpu_hotplug {
4276     my ($vmid, $conf, $vcpus) = @_;
4277
4278     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4279
4280     my $sockets = 1;
4281     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4282     $sockets = $conf->{sockets} if  $conf->{sockets};
4283     my $cores = $conf->{cores} || 1;
4284     my $maxcpus = $sockets * $cores;
4285
4286     $vcpus = $maxcpus if !$vcpus;
4287
4288     die "you can't add more vcpus than maxcpus\n"
4289         if $vcpus > $maxcpus;
4290
4291     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4292
4293     if ($vcpus < $currentvcpus) {
4294
4295         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4296
4297             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4298                 qemu_devicedel($vmid, "cpu$i");
4299                 my $retry = 0;
4300                 my $currentrunningvcpus = undef;
4301                 while (1) {
4302                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4303                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4304                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4305                     $retry++;
4306                     sleep 1;
4307                 }
4308                 #update conf after each succesfull cpu unplug
4309                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4310                 PVE::QemuConfig->write_config($vmid, $conf);
4311             }
4312         } else {
4313             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4314         }
4315
4316         return;
4317     }
4318
4319     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4320     die "vcpus in running vm does not match its configuration\n"
4321         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4322
4323     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4324
4325         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4326             my $cpustr = print_cpu_device($conf, $i);
4327             qemu_deviceadd($vmid, $cpustr);
4328
4329             my $retry = 0;
4330             my $currentrunningvcpus = undef;
4331             while (1) {
4332                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4333                 last if scalar(@{$currentrunningvcpus}) == $i;
4334                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4335                 sleep 1;
4336                 $retry++;
4337             }
4338             #update conf after each succesfull cpu hotplug
4339             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4340             PVE::QemuConfig->write_config($vmid, $conf);
4341         }
4342     } else {
4343
4344         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4345             mon_cmd($vmid, "cpu-add", id => int($i));
4346         }
4347     }
4348 }
4349
4350 sub qemu_block_set_io_throttle {
4351     my ($vmid, $deviceid,
4352         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4353         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4354         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4355         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4356
4357     return if !check_running($vmid) ;
4358
4359     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4360         bps => int($bps),
4361         bps_rd => int($bps_rd),
4362         bps_wr => int($bps_wr),
4363         iops => int($iops),
4364         iops_rd => int($iops_rd),
4365         iops_wr => int($iops_wr),
4366         bps_max => int($bps_max),
4367         bps_rd_max => int($bps_rd_max),
4368         bps_wr_max => int($bps_wr_max),
4369         iops_max => int($iops_max),
4370         iops_rd_max => int($iops_rd_max),
4371         iops_wr_max => int($iops_wr_max),
4372         bps_max_length => int($bps_max_length),
4373         bps_rd_max_length => int($bps_rd_max_length),
4374         bps_wr_max_length => int($bps_wr_max_length),
4375         iops_max_length => int($iops_max_length),
4376         iops_rd_max_length => int($iops_rd_max_length),
4377         iops_wr_max_length => int($iops_wr_max_length),
4378     );
4379
4380 }
4381
4382 sub qemu_block_resize {
4383     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4384
4385     my $running = check_running($vmid);
4386
4387     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4388
4389     return if !$running;
4390
4391     my $padding = (1024 - $size % 1024) % 1024;
4392     $size = $size + $padding;
4393
4394     mon_cmd(
4395         $vmid,
4396         "block_resize",
4397         device => $deviceid,
4398         size => int($size),
4399         timeout => 60,
4400     );
4401 }
4402
4403 sub qemu_volume_snapshot {
4404     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4405
4406     my $running = check_running($vmid);
4407
4408     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4409         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4410     } else {
4411         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4412     }
4413 }
4414
4415 sub qemu_volume_snapshot_delete {
4416     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4417
4418     my $running = check_running($vmid);
4419
4420     if($running) {
4421
4422         $running = undef;
4423         my $conf = PVE::QemuConfig->load_config($vmid);
4424         PVE::QemuConfig->foreach_volume($conf, sub {
4425             my ($ds, $drive) = @_;
4426             $running = 1 if $drive->{file} eq $volid;
4427         });
4428     }
4429
4430     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4431         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4432     } else {
4433         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4434     }
4435 }
4436
4437 sub set_migration_caps {
4438     my ($vmid, $savevm) = @_;
4439
4440     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4441
4442     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4443     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4444
4445     my $cap_ref = [];
4446
4447     my $enabled_cap = {
4448         "auto-converge" => 1,
4449         "xbzrle" => 1,
4450         "x-rdma-pin-all" => 0,
4451         "zero-blocks" => 0,
4452         "compress" => 0,
4453         "dirty-bitmaps" => $dirty_bitmaps,
4454     };
4455
4456     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4457
4458     for my $supported_capability (@$supported_capabilities) {
4459         push @$cap_ref, {
4460             capability => $supported_capability->{capability},
4461             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4462         };
4463     }
4464
4465     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4466 }
4467
4468 sub foreach_volid {
4469     my ($conf, $func, @param) = @_;
4470
4471     my $volhash = {};
4472
4473     my $test_volid = sub {
4474         my ($key, $drive, $snapname) = @_;
4475
4476         my $volid = $drive->{file};
4477         return if !$volid;
4478
4479         $volhash->{$volid}->{cdrom} //= 1;
4480         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4481
4482         my $replicate = $drive->{replicate} // 1;
4483         $volhash->{$volid}->{replicate} //= 0;
4484         $volhash->{$volid}->{replicate} = 1 if $replicate;
4485
4486         $volhash->{$volid}->{shared} //= 0;
4487         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4488
4489         $volhash->{$volid}->{referenced_in_config} //= 0;
4490         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4491
4492         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4493             if defined($snapname);
4494
4495         my $size = $drive->{size};
4496         $volhash->{$volid}->{size} //= $size if $size;
4497
4498         $volhash->{$volid}->{is_vmstate} //= 0;
4499         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4500
4501         $volhash->{$volid}->{is_unused} //= 0;
4502         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4503
4504         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4505     };
4506
4507     my $include_opts = {
4508         extra_keys => ['vmstate'],
4509         include_unused => 1,
4510     };
4511
4512     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4513     foreach my $snapname (keys %{$conf->{snapshots}}) {
4514         my $snap = $conf->{snapshots}->{$snapname};
4515         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4516     }
4517
4518     foreach my $volid (keys %$volhash) {
4519         &$func($volid, $volhash->{$volid}, @param);
4520     }
4521 }
4522
4523 my $fast_plug_option = {
4524     'lock' => 1,
4525     'name' => 1,
4526     'onboot' => 1,
4527     'shares' => 1,
4528     'startup' => 1,
4529     'description' => 1,
4530     'protection' => 1,
4531     'vmstatestorage' => 1,
4532     'hookscript' => 1,
4533     'tags' => 1,
4534 };
4535
4536 # hotplug changes in [PENDING]
4537 # $selection hash can be used to only apply specified options, for
4538 # example: { cores => 1 } (only apply changed 'cores')
4539 # $errors ref is used to return error messages
4540 sub vmconfig_hotplug_pending {
4541     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4542
4543     my $defaults = load_defaults();
4544     my $arch = get_vm_arch($conf);
4545     my $machine_type = get_vm_machine($conf, undef, $arch);
4546
4547     # commit values which do not have any impact on running VM first
4548     # Note: those option cannot raise errors, we we do not care about
4549     # $selection and always apply them.
4550
4551     my $add_error = sub {
4552         my ($opt, $msg) = @_;
4553         $errors->{$opt} = "hotplug problem - $msg";
4554     };
4555
4556     my $changes = 0;
4557     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4558         if ($fast_plug_option->{$opt}) {
4559             $conf->{$opt} = $conf->{pending}->{$opt};
4560             delete $conf->{pending}->{$opt};
4561             $changes = 1;
4562         }
4563     }
4564
4565     if ($changes) {
4566         PVE::QemuConfig->write_config($vmid, $conf);
4567     }
4568
4569     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4570
4571     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4572     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4573     foreach my $opt (sort keys %$pending_delete_hash) {
4574         next if $selection && !$selection->{$opt};
4575         my $force = $pending_delete_hash->{$opt}->{force};
4576         eval {
4577             if ($opt eq 'hotplug') {
4578                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4579             } elsif ($opt eq 'tablet') {
4580                 die "skip\n" if !$hotplug_features->{usb};
4581                 if ($defaults->{tablet}) {
4582                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4583                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4584                         if $arch eq 'aarch64';
4585                 } else {
4586                     vm_deviceunplug($vmid, $conf, 'tablet');
4587                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4588                 }
4589             } elsif ($opt =~ m/^usb\d+/) {
4590                 die "skip\n";
4591                 # since we cannot reliably hot unplug usb devices we are disabling it
4592                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4593                 #vm_deviceunplug($vmid, $conf, $opt);
4594             } elsif ($opt eq 'vcpus') {
4595                 die "skip\n" if !$hotplug_features->{cpu};
4596                 qemu_cpu_hotplug($vmid, $conf, undef);
4597             } elsif ($opt eq 'balloon') {
4598                 # enable balloon device is not hotpluggable
4599                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4600                 # here we reset the ballooning value to memory
4601                 my $balloon = $conf->{memory} || $defaults->{memory};
4602                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4603             } elsif ($fast_plug_option->{$opt}) {
4604                 # do nothing
4605             } elsif ($opt =~ m/^net(\d+)$/) {
4606                 die "skip\n" if !$hotplug_features->{network};
4607                 vm_deviceunplug($vmid, $conf, $opt);
4608             } elsif (is_valid_drivename($opt)) {
4609                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4610                 vm_deviceunplug($vmid, $conf, $opt);
4611                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4612             } elsif ($opt =~ m/^memory$/) {
4613                 die "skip\n" if !$hotplug_features->{memory};
4614                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4615             } elsif ($opt eq 'cpuunits') {
4616                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4617             } elsif ($opt eq 'cpulimit') {
4618                 $cgroup->change_cpu_quota(-1, 100000);
4619             } else {
4620                 die "skip\n";
4621             }
4622         };
4623         if (my $err = $@) {
4624             &$add_error($opt, $err) if $err ne "skip\n";
4625         } else {
4626             delete $conf->{$opt};
4627             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4628         }
4629     }
4630
4631     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4632     $apply_pending_cloudinit = sub {
4633         return if $apply_pending_cloudinit_done; # once is enough
4634         $apply_pending_cloudinit_done = 1; # once is enough
4635
4636         my ($key, $value) = @_;
4637
4638         my @cloudinit_opts = keys %$confdesc_cloudinit;
4639         foreach my $opt (keys %{$conf->{pending}}) {
4640             next if !grep { $_ eq $opt } @cloudinit_opts;
4641             $conf->{$opt} = delete $conf->{pending}->{$opt};
4642         }
4643
4644         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4645         foreach my $opt (sort keys %$pending_delete_hash) {
4646             next if !grep { $_ eq $opt } @cloudinit_opts;
4647             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4648             delete $conf->{$opt};
4649         }
4650
4651         my $new_conf = { %$conf };
4652         $new_conf->{$key} = $value;
4653         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4654     };
4655
4656     foreach my $opt (keys %{$conf->{pending}}) {
4657         next if $selection && !$selection->{$opt};
4658         my $value = $conf->{pending}->{$opt};
4659         eval {
4660             if ($opt eq 'hotplug') {
4661                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4662             } elsif ($opt eq 'tablet') {
4663                 die "skip\n" if !$hotplug_features->{usb};
4664                 if ($value == 1) {
4665                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4666                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4667                         if $arch eq 'aarch64';
4668                 } elsif ($value == 0) {
4669                     vm_deviceunplug($vmid, $conf, 'tablet');
4670                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4671                 }
4672             } elsif ($opt =~ m/^usb\d+$/) {
4673                 die "skip\n";
4674                 # since we cannot reliably hot unplug usb devices we disable it for now
4675                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4676                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4677                 #die "skip\n" if !$d;
4678                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4679             } elsif ($opt eq 'vcpus') {
4680                 die "skip\n" if !$hotplug_features->{cpu};
4681                 qemu_cpu_hotplug($vmid, $conf, $value);
4682             } elsif ($opt eq 'balloon') {
4683                 # enable/disable balloning device is not hotpluggable
4684                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4685                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4686                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4687
4688                 # allow manual ballooning if shares is set to zero
4689                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4690                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4691                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4692                 }
4693             } elsif ($opt =~ m/^net(\d+)$/) {
4694                 # some changes can be done without hotplug
4695                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4696                                     $vmid, $opt, $value, $arch, $machine_type);
4697             } elsif (is_valid_drivename($opt)) {
4698                 die "skip\n" if $opt eq 'efidisk0';
4699                 # some changes can be done without hotplug
4700                 my $drive = parse_drive($opt, $value);
4701                 if (drive_is_cloudinit($drive)) {
4702                     &$apply_pending_cloudinit($opt, $value);
4703                 }
4704                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4705                                      $vmid, $opt, $value, $arch, $machine_type);
4706             } elsif ($opt =~ m/^memory$/) { #dimms
4707                 die "skip\n" if !$hotplug_features->{memory};
4708                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4709             } elsif ($opt eq 'cpuunits') {
4710                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4711             } elsif ($opt eq 'cpulimit') {
4712                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4713                 $cgroup->change_cpu_quota($cpulimit, 100000);
4714             } else {
4715                 die "skip\n";  # skip non-hot-pluggable options
4716             }
4717         };
4718         if (my $err = $@) {
4719             &$add_error($opt, $err) if $err ne "skip\n";
4720         } else {
4721             $conf->{$opt} = $value;
4722             delete $conf->{pending}->{$opt};
4723         }
4724     }
4725
4726     PVE::QemuConfig->write_config($vmid, $conf);
4727 }
4728
4729 sub try_deallocate_drive {
4730     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4731
4732     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4733         my $volid = $drive->{file};
4734         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4735             my $sid = PVE::Storage::parse_volume_id($volid);
4736             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4737
4738             # check if the disk is really unused
4739             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4740                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4741             PVE::Storage::vdisk_free($storecfg, $volid);
4742             return 1;
4743         } else {
4744             # If vm is not owner of this disk remove from config
4745             return 1;
4746         }
4747     }
4748
4749     return;
4750 }
4751
4752 sub vmconfig_delete_or_detach_drive {
4753     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4754
4755     my $drive = parse_drive($opt, $conf->{$opt});
4756
4757     my $rpcenv = PVE::RPCEnvironment::get();
4758     my $authuser = $rpcenv->get_user();
4759
4760     if ($force) {
4761         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4762         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4763     } else {
4764         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4765     }
4766 }
4767
4768
4769
4770 sub vmconfig_apply_pending {
4771     my ($vmid, $conf, $storecfg, $errors) = @_;
4772
4773     my $add_apply_error = sub {
4774         my ($opt, $msg) = @_;
4775         my $err_msg = "unable to apply pending change $opt : $msg";
4776         $errors->{$opt} = $err_msg;
4777         warn $err_msg;
4778     };
4779
4780     # cold plug
4781
4782     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4783     foreach my $opt (sort keys %$pending_delete_hash) {
4784         my $force = $pending_delete_hash->{$opt}->{force};
4785         eval {
4786             if ($opt =~ m/^unused/) {
4787                 die "internal error";
4788             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4789                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4790             }
4791         };
4792         if (my $err = $@) {
4793             $add_apply_error->($opt, $err);
4794         } else {
4795             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4796             delete $conf->{$opt};
4797         }
4798     }
4799
4800     PVE::QemuConfig->cleanup_pending($conf);
4801
4802     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4803         next if $opt eq 'delete'; # just to be sure
4804         eval {
4805             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4806                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4807             }
4808         };
4809         if (my $err = $@) {
4810             $add_apply_error->($opt, $err);
4811         } else {
4812             $conf->{$opt} = delete $conf->{pending}->{$opt};
4813         }
4814     }
4815
4816     # write all changes at once to avoid unnecessary i/o
4817     PVE::QemuConfig->write_config($vmid, $conf);
4818 }
4819
4820 sub vmconfig_update_net {
4821     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4822
4823     my $newnet = parse_net($value);
4824
4825     if ($conf->{$opt}) {
4826         my $oldnet = parse_net($conf->{$opt});
4827
4828         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4829             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4830             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4831             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4832
4833             # for non online change, we try to hot-unplug
4834             die "skip\n" if !$hotplug;
4835             vm_deviceunplug($vmid, $conf, $opt);
4836         } else {
4837
4838             die "internal error" if $opt !~ m/net(\d+)/;
4839             my $iface = "tap${vmid}i$1";
4840
4841             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4842                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4843                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4844                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4845                 PVE::Network::tap_unplug($iface);
4846
4847                 if ($have_sdn) {
4848                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4849                 } else {
4850                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4851                 }
4852             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4853                 # Rate can be applied on its own but any change above needs to
4854                 # include the rate in tap_plug since OVS resets everything.
4855                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4856             }
4857
4858             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4859                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4860             }
4861
4862             return 1;
4863         }
4864     }
4865
4866     if ($hotplug) {
4867         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4868     } else {
4869         die "skip\n";
4870     }
4871 }
4872
4873 sub vmconfig_update_disk {
4874     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4875
4876     my $drive = parse_drive($opt, $value);
4877
4878     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4879         my $media = $drive->{media} || 'disk';
4880         my $oldmedia = $old_drive->{media} || 'disk';
4881         die "unable to change media type\n" if $media ne $oldmedia;
4882
4883         if (!drive_is_cdrom($old_drive)) {
4884
4885             if ($drive->{file} ne $old_drive->{file}) {
4886
4887                 die "skip\n" if !$hotplug;
4888
4889                 # unplug and register as unused
4890                 vm_deviceunplug($vmid, $conf, $opt);
4891                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4892
4893             } else {
4894                 # update existing disk
4895
4896                 # skip non hotpluggable value
4897                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4898                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4899                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4900                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4901                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4902                     die "skip\n";
4903                 }
4904
4905                 # apply throttle
4906                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4907                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4908                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4909                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4910                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4911                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4912                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4913                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4914                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4915                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4916                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4917                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4918                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4919                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4920                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4921                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4922                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4923                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4924
4925                     qemu_block_set_io_throttle(
4926                         $vmid,"drive-$opt",
4927                         ($drive->{mbps} || 0)*1024*1024,
4928                         ($drive->{mbps_rd} || 0)*1024*1024,
4929                         ($drive->{mbps_wr} || 0)*1024*1024,
4930                          $drive->{iops} || 0,
4931                          $drive->{iops_rd} || 0,
4932                          $drive->{iops_wr} || 0,
4933                         ($drive->{mbps_max} || 0)*1024*1024,
4934                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4935                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4936                          $drive->{iops_max} || 0,
4937                          $drive->{iops_rd_max} || 0,
4938                          $drive->{iops_wr_max} || 0,
4939                          $drive->{bps_max_length} || 1,
4940                          $drive->{bps_rd_max_length} || 1,
4941                          $drive->{bps_wr_max_length} || 1,
4942                          $drive->{iops_max_length} || 1,
4943                          $drive->{iops_rd_max_length} || 1,
4944                          $drive->{iops_wr_max_length} || 1,
4945                     );
4946
4947                 }
4948
4949                 return 1;
4950             }
4951
4952         } else { # cdrom
4953
4954             if ($drive->{file} eq 'none') {
4955                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4956                 if (drive_is_cloudinit($old_drive)) {
4957                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4958                 }
4959             } else {
4960                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4961
4962                 # force eject if locked
4963                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4964
4965                 if ($path) {
4966                     mon_cmd($vmid, "blockdev-change-medium",
4967                         id => "$opt", filename => "$path");
4968                 }
4969             }
4970
4971             return 1;
4972         }
4973     }
4974
4975     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4976     # hotplug new disks
4977     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4978     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4979 }
4980
4981 # called in locked context by incoming migration
4982 sub vm_migrate_get_nbd_disks {
4983     my ($storecfg, $conf, $replicated_volumes) = @_;
4984
4985     my $local_volumes = {};
4986     PVE::QemuConfig->foreach_volume($conf, sub {
4987         my ($ds, $drive) = @_;
4988
4989         return if drive_is_cdrom($drive);
4990
4991         my $volid = $drive->{file};
4992
4993         return if !$volid;
4994
4995         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4996
4997         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4998         return if $scfg->{shared};
4999
5000         # replicated disks re-use existing state via bitmap
5001         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5002         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5003     });
5004     return $local_volumes;
5005 }
5006
5007 # called in locked context by incoming migration
5008 sub vm_migrate_alloc_nbd_disks {
5009     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5010
5011     my $format = undef;
5012
5013     my $nbd = {};
5014     foreach my $opt (sort keys %$source_volumes) {
5015         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5016
5017         if ($use_existing) {
5018             $nbd->{$opt}->{drivestr} = print_drive($drive);
5019             $nbd->{$opt}->{volid} = $volid;
5020             $nbd->{$opt}->{replicated} = 1;
5021             next;
5022         }
5023
5024         # If a remote storage is specified and the format of the original
5025         # volume is not available there, fall back to the default format.
5026         # Otherwise use the same format as the original.
5027         if (!$storagemap->{identity}) {
5028             $storeid = map_storage($storagemap, $storeid);
5029             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5030             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5031             my $fileFormat = qemu_img_format($scfg, $volname);
5032             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5033         } else {
5034             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5035             $format = qemu_img_format($scfg, $volname);
5036         }
5037
5038         my $size = $drive->{size} / 1024;
5039         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5040         my $newdrive = $drive;
5041         $newdrive->{format} = $format;
5042         $newdrive->{file} = $newvolid;
5043         my $drivestr = print_drive($newdrive);
5044         $nbd->{$opt}->{drivestr} = $drivestr;
5045         $nbd->{$opt}->{volid} = $newvolid;
5046     }
5047
5048     return $nbd;
5049 }
5050
5051 # see vm_start_nolock for parameters, additionally:
5052 # migrate_opts:
5053 #   storagemap = parsed storage map for allocating NBD disks
5054 sub vm_start {
5055     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5056
5057     return PVE::QemuConfig->lock_config($vmid, sub {
5058         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5059
5060         die "you can't start a vm if it's a template\n"
5061             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5062
5063         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5064         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5065
5066         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5067
5068         if ($has_backup_lock && $running) {
5069             # a backup is currently running, attempt to start the guest in the
5070             # existing QEMU instance
5071             return vm_resume($vmid);
5072         }
5073
5074         PVE::QemuConfig->check_lock($conf)
5075             if !($params->{skiplock} || $has_suspended_lock);
5076
5077         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5078
5079         die "VM $vmid already running\n" if $running;
5080
5081         if (my $storagemap = $migrate_opts->{storagemap}) {
5082             my $replicated = $migrate_opts->{replicated_volumes};
5083             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5084             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5085
5086             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5087                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5088             }
5089         }
5090
5091         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5092     });
5093 }
5094
5095
5096 # params:
5097 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5098 #   skiplock => 0/1, skip checking for config lock
5099 #   skiptemplate => 0/1, skip checking whether VM is template
5100 #   forcemachine => to force Qemu machine (rollback/migration)
5101 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5102 #   timeout => in seconds
5103 #   paused => start VM in paused state (backup)
5104 #   resume => resume from hibernation
5105 #   pbs-backing => {
5106 #      sata0 => {
5107 #         repository
5108 #         snapshot
5109 #         keyfile
5110 #         archive
5111 #      },
5112 #      virtio2 => ...
5113 #   }
5114 # migrate_opts:
5115 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5116 #   migratedfrom => source node
5117 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5118 #   network => CIDR of migration network
5119 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5120 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5121 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5122 sub vm_start_nolock {
5123     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5124
5125     my $statefile = $params->{statefile};
5126     my $resume = $params->{resume};
5127
5128     my $migratedfrom = $migrate_opts->{migratedfrom};
5129     my $migration_type = $migrate_opts->{type};
5130
5131     my $res = {};
5132
5133     # clean up leftover reboot request files
5134     eval { clear_reboot_request($vmid); };
5135     warn $@ if $@;
5136
5137     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5138         vmconfig_apply_pending($vmid, $conf, $storecfg);
5139         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5140     }
5141
5142     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5143
5144     my $defaults = load_defaults();
5145
5146     # set environment variable useful inside network script
5147     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5148
5149     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5150
5151     my $forcemachine = $params->{forcemachine};
5152     my $forcecpu = $params->{forcecpu};
5153     if ($resume) {
5154         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5155         $forcemachine = $conf->{runningmachine};
5156         $forcecpu = $conf->{runningcpu};
5157         print "Resuming suspended VM\n";
5158     }
5159
5160     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5161         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5162
5163     my $migration_ip;
5164     my $get_migration_ip = sub {
5165         my ($nodename) = @_;
5166
5167         return $migration_ip if defined($migration_ip);
5168
5169         my $cidr = $migrate_opts->{network};
5170
5171         if (!defined($cidr)) {
5172             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5173             $cidr = $dc_conf->{migration}->{network};
5174         }
5175
5176         if (defined($cidr)) {
5177             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5178
5179             die "could not get IP: no address configured on local " .
5180                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5181
5182             die "could not get IP: multiple addresses configured on local " .
5183                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5184
5185             $migration_ip = @$ips[0];
5186         }
5187
5188         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5189             if !defined($migration_ip);
5190
5191         return $migration_ip;
5192     };
5193
5194     my $migrate_uri;
5195     if ($statefile) {
5196         if ($statefile eq 'tcp') {
5197             my $localip = "localhost";
5198             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5199             my $nodename = nodename();
5200
5201             if (!defined($migration_type)) {
5202                 if (defined($datacenterconf->{migration}->{type})) {
5203                     $migration_type = $datacenterconf->{migration}->{type};
5204                 } else {
5205                     $migration_type = 'secure';
5206                 }
5207             }
5208
5209             if ($migration_type eq 'insecure') {
5210                 $localip = $get_migration_ip->($nodename);
5211                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5212             }
5213
5214             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5215             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5216             $migrate_uri = "tcp:${localip}:${migrate_port}";
5217             push @$cmd, '-incoming', $migrate_uri;
5218             push @$cmd, '-S';
5219
5220         } elsif ($statefile eq 'unix') {
5221             # should be default for secure migrations as a ssh TCP forward
5222             # tunnel is not deterministic reliable ready and fails regurarly
5223             # to set up in time, so use UNIX socket forwards
5224             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5225             unlink $socket_addr;
5226
5227             $migrate_uri = "unix:$socket_addr";
5228
5229             push @$cmd, '-incoming', $migrate_uri;
5230             push @$cmd, '-S';
5231
5232         } elsif (-e $statefile) {
5233             push @$cmd, '-loadstate', $statefile;
5234         } else {
5235             my $statepath = PVE::Storage::path($storecfg, $statefile);
5236             push @$vollist, $statefile;
5237             push @$cmd, '-loadstate', $statepath;
5238         }
5239     } elsif ($params->{paused}) {
5240         push @$cmd, '-S';
5241     }
5242
5243     # host pci devices
5244     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5245       my $d = parse_hostpci($conf->{"hostpci$i"});
5246       next if !$d;
5247       my $pcidevices = $d->{pciid};
5248       foreach my $pcidevice (@$pcidevices) {
5249             my $pciid = $pcidevice->{id};
5250
5251             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5252             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5253             die "no pci device info for device '$pciid'\n" if !$info;
5254
5255             if ($d->{mdev}) {
5256                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5257                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5258             } else {
5259                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5260                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5261                 die "can't reset PCI device '$pciid'\n"
5262                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5263             }
5264       }
5265     }
5266
5267     PVE::Storage::activate_volumes($storecfg, $vollist);
5268
5269     eval {
5270         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5271             outfunc => sub {}, errfunc => sub {});
5272     };
5273     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5274     # timeout should be more than enough here...
5275     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5276
5277     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5278                                               : $defaults->{cpuunits};
5279
5280     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5281     my %run_params = (
5282         timeout => $statefile ? undef : $start_timeout,
5283         umask => 0077,
5284         noerr => 1,
5285     );
5286
5287     # when migrating, prefix QEMU output so other side can pick up any
5288     # errors that might occur and show the user
5289     if ($migratedfrom) {
5290         $run_params{quiet} = 1;
5291         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5292     }
5293
5294     my %properties = (
5295         Slice => 'qemu.slice',
5296         KillMode => 'process',
5297         SendSIGKILL => 0,
5298         TimeoutStopUSec => ULONG_MAX, # infinity
5299     );
5300
5301     if (PVE::CGroup::cgroup_mode() == 2) {
5302         $properties{CPUWeight} = $cpuunits;
5303     } else {
5304         $properties{CPUShares} = $cpuunits;
5305     }
5306
5307     if (my $cpulimit = $conf->{cpulimit}) {
5308         $properties{CPUQuota} = int($cpulimit * 100);
5309     }
5310     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5311
5312     my $run_qemu = sub {
5313         PVE::Tools::run_fork sub {
5314             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5315
5316             my $exitcode = run_command($cmd, %run_params);
5317             die "QEMU exited with code $exitcode\n" if $exitcode;
5318         };
5319     };
5320
5321     if ($conf->{hugepages}) {
5322
5323         my $code = sub {
5324             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5325             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5326
5327             PVE::QemuServer::Memory::hugepages_mount();
5328             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5329
5330             eval { $run_qemu->() };
5331             if (my $err = $@) {
5332                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5333                     if !$conf->{keephugepages};
5334                 die $err;
5335             }
5336
5337             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5338                 if !$conf->{keephugepages};
5339         };
5340         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5341
5342     } else {
5343         eval { $run_qemu->() };
5344     }
5345
5346     if (my $err = $@) {
5347         # deactivate volumes if start fails
5348         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5349         die "start failed: $err";
5350     }
5351
5352     print "migration listens on $migrate_uri\n" if $migrate_uri;
5353     $res->{migrate_uri} = $migrate_uri;
5354
5355     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5356         eval { mon_cmd($vmid, "cont"); };
5357         warn $@ if $@;
5358     }
5359
5360     #start nbd server for storage migration
5361     if (my $nbd = $migrate_opts->{nbd}) {
5362         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5363
5364         my $migrate_storage_uri;
5365         # nbd_protocol_version > 0 for unix socket support
5366         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5367             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5368             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5369             $migrate_storage_uri = "nbd:unix:$socket_path";
5370         } else {
5371             my $nodename = nodename();
5372             my $localip = $get_migration_ip->($nodename);
5373             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5374             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5375
5376             mon_cmd($vmid, "nbd-server-start", addr => {
5377                 type => 'inet',
5378                 data => {
5379                     host => "${localip}",
5380                     port => "${storage_migrate_port}",
5381                 },
5382             });
5383             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5384             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5385         }
5386
5387         $res->{migrate_storage_uri} = $migrate_storage_uri;
5388
5389         foreach my $opt (sort keys %$nbd) {
5390             my $drivestr = $nbd->{$opt}->{drivestr};
5391             my $volid = $nbd->{$opt}->{volid};
5392             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5393             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5394             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5395             print "re-using replicated volume: $opt - $volid\n"
5396                 if $nbd->{$opt}->{replicated};
5397
5398             $res->{drives}->{$opt} = $nbd->{$opt};
5399             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5400         }
5401     }
5402
5403     if ($migratedfrom) {
5404         eval {
5405             set_migration_caps($vmid);
5406         };
5407         warn $@ if $@;
5408
5409         if ($spice_port) {
5410             print "spice listens on port $spice_port\n";
5411             $res->{spice_port} = $spice_port;
5412             if ($migrate_opts->{spice_ticket}) {
5413                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5414                     $migrate_opts->{spice_ticket});
5415                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5416             }
5417         }
5418
5419     } else {
5420         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5421             if !$statefile && $conf->{balloon};
5422
5423         foreach my $opt (keys %$conf) {
5424             next if $opt !~  m/^net\d+$/;
5425             my $nicconf = parse_net($conf->{$opt});
5426             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5427         }
5428     }
5429
5430     mon_cmd($vmid, 'qom-set',
5431                 path => "machine/peripheral/balloon0",
5432                 property => "guest-stats-polling-interval",
5433                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5434
5435     if ($resume) {
5436         print "Resumed VM, removing state\n";
5437         if (my $vmstate = $conf->{vmstate}) {
5438             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5439             PVE::Storage::vdisk_free($storecfg, $vmstate);
5440         }
5441         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5442         PVE::QemuConfig->write_config($vmid, $conf);
5443     }
5444
5445     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5446
5447     return $res;
5448 }
5449
5450 sub vm_commandline {
5451     my ($storecfg, $vmid, $snapname) = @_;
5452
5453     my $conf = PVE::QemuConfig->load_config($vmid);
5454     my $forcemachine;
5455     my $forcecpu;
5456
5457     if ($snapname) {
5458         my $snapshot = $conf->{snapshots}->{$snapname};
5459         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5460
5461         # check for machine or CPU overrides in snapshot
5462         $forcemachine = $snapshot->{runningmachine};
5463         $forcecpu = $snapshot->{runningcpu};
5464
5465         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5466
5467         $conf = $snapshot;
5468     }
5469
5470     my $defaults = load_defaults();
5471
5472     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5473         $forcemachine, $forcecpu);
5474
5475     return PVE::Tools::cmd2string($cmd);
5476 }
5477
5478 sub vm_reset {
5479     my ($vmid, $skiplock) = @_;
5480
5481     PVE::QemuConfig->lock_config($vmid, sub {
5482
5483         my $conf = PVE::QemuConfig->load_config($vmid);
5484
5485         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5486
5487         mon_cmd($vmid, "system_reset");
5488     });
5489 }
5490
5491 sub get_vm_volumes {
5492     my ($conf) = @_;
5493
5494     my $vollist = [];
5495     foreach_volid($conf, sub {
5496         my ($volid, $attr) = @_;
5497
5498         return if $volid =~ m|^/|;
5499
5500         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5501         return if !$sid;
5502
5503         push @$vollist, $volid;
5504     });
5505
5506     return $vollist;
5507 }
5508
5509 sub vm_stop_cleanup {
5510     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5511
5512     eval {
5513
5514         if (!$keepActive) {
5515             my $vollist = get_vm_volumes($conf);
5516             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5517         }
5518
5519         foreach my $ext (qw(mon qmp pid vnc qga)) {
5520             unlink "/var/run/qemu-server/${vmid}.$ext";
5521         }
5522
5523         if ($conf->{ivshmem}) {
5524             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5525             # just delete it for now, VMs which have this already open do not
5526             # are affected, but new VMs will get a separated one. If this
5527             # becomes an issue we either add some sort of ref-counting or just
5528             # add a "don't delete on stop" flag to the ivshmem format.
5529             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5530         }
5531
5532         foreach my $key (keys %$conf) {
5533             next if $key !~ m/^hostpci(\d+)$/;
5534             my $hostpciindex = $1;
5535             my $d = parse_hostpci($conf->{$key});
5536             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5537
5538             foreach my $pci (@{$d->{pciid}}) {
5539                 my $pciid = $pci->{id};
5540                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5541             }
5542         }
5543
5544         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5545     };
5546     warn $@ if $@; # avoid errors - just warn
5547 }
5548
5549 # call only in locked context
5550 sub _do_vm_stop {
5551     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5552
5553     my $pid = check_running($vmid, $nocheck);
5554     return if !$pid;
5555
5556     my $conf;
5557     if (!$nocheck) {
5558         $conf = PVE::QemuConfig->load_config($vmid);
5559         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5560         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5561             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5562             $timeout = $opts->{down} if $opts->{down};
5563         }
5564         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5565     }
5566
5567     eval {
5568         if ($shutdown) {
5569             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5570                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5571             } else {
5572                 mon_cmd($vmid, "system_powerdown");
5573             }
5574         } else {
5575             mon_cmd($vmid, "quit");
5576         }
5577     };
5578     my $err = $@;
5579
5580     if (!$err) {
5581         $timeout = 60 if !defined($timeout);
5582
5583         my $count = 0;
5584         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5585             $count++;
5586             sleep 1;
5587         }
5588
5589         if ($count >= $timeout) {
5590             if ($force) {
5591                 warn "VM still running - terminating now with SIGTERM\n";
5592                 kill 15, $pid;
5593             } else {
5594                 die "VM quit/powerdown failed - got timeout\n";
5595             }
5596         } else {
5597             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5598             return;
5599         }
5600     } else {
5601        if (!check_running($vmid, $nocheck)) {
5602             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5603             return;
5604        }
5605        if ($force) {
5606             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5607             kill 15, $pid;
5608         } else {
5609             die "VM quit/powerdown failed\n";
5610         }
5611     }
5612
5613     # wait again
5614     $timeout = 10;
5615
5616     my $count = 0;
5617     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5618         $count++;
5619         sleep 1;
5620     }
5621
5622     if ($count >= $timeout) {
5623         warn "VM still running - terminating now with SIGKILL\n";
5624         kill 9, $pid;
5625         sleep 1;
5626     }
5627
5628     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5629 }
5630
5631 # Note: use $nocheck to skip tests if VM configuration file exists.
5632 # We need that when migration VMs to other nodes (files already moved)
5633 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5634 sub vm_stop {
5635     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5636
5637     $force = 1 if !defined($force) && !$shutdown;
5638
5639     if ($migratedfrom){
5640         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5641         kill 15, $pid if $pid;
5642         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5643         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5644         return;
5645     }
5646
5647     PVE::QemuConfig->lock_config($vmid, sub {
5648         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5649    });
5650 }
5651
5652 sub vm_reboot {
5653     my ($vmid, $timeout) = @_;
5654
5655     PVE::QemuConfig->lock_config($vmid, sub {
5656         eval {
5657
5658             # only reboot if running, as qmeventd starts it again on a stop event
5659             return if !check_running($vmid);
5660
5661             create_reboot_request($vmid);
5662
5663             my $storecfg = PVE::Storage::config();
5664             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5665
5666         };
5667         if (my $err = $@) {
5668             # avoid that the next normal shutdown will be confused for a reboot
5669             clear_reboot_request($vmid);
5670             die $err;
5671         }
5672    });
5673 }
5674
5675 # note: if using the statestorage parameter, the caller has to check privileges
5676 sub vm_suspend {
5677     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5678
5679     my $conf;
5680     my $path;
5681     my $storecfg;
5682     my $vmstate;
5683
5684     PVE::QemuConfig->lock_config($vmid, sub {
5685
5686         $conf = PVE::QemuConfig->load_config($vmid);
5687
5688         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5689         PVE::QemuConfig->check_lock($conf)
5690             if !($skiplock || $is_backing_up);
5691
5692         die "cannot suspend to disk during backup\n"
5693             if $is_backing_up && $includestate;
5694
5695         if ($includestate) {
5696             $conf->{lock} = 'suspending';
5697             my $date = strftime("%Y-%m-%d", localtime(time()));
5698             $storecfg = PVE::Storage::config();
5699             if (!$statestorage) {
5700                 $statestorage = find_vmstate_storage($conf, $storecfg);
5701                 # check permissions for the storage
5702                 my $rpcenv = PVE::RPCEnvironment::get();
5703                 if ($rpcenv->{type} ne 'cli') {
5704                     my $authuser = $rpcenv->get_user();
5705                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5706                 }
5707             }
5708
5709
5710             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5711                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5712             $path = PVE::Storage::path($storecfg, $vmstate);
5713             PVE::QemuConfig->write_config($vmid, $conf);
5714         } else {
5715             mon_cmd($vmid, "stop");
5716         }
5717     });
5718
5719     if ($includestate) {
5720         # save vm state
5721         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5722
5723         eval {
5724             set_migration_caps($vmid, 1);
5725             mon_cmd($vmid, "savevm-start", statefile => $path);
5726             for(;;) {
5727                 my $state = mon_cmd($vmid, "query-savevm");
5728                 if (!$state->{status}) {
5729                     die "savevm not active\n";
5730                 } elsif ($state->{status} eq 'active') {
5731                     sleep(1);
5732                     next;
5733                 } elsif ($state->{status} eq 'completed') {
5734                     print "State saved, quitting\n";
5735                     last;
5736                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5737                     die "query-savevm failed with error '$state->{error}'\n"
5738                 } else {
5739                     die "query-savevm returned status '$state->{status}'\n";
5740                 }
5741             }
5742         };
5743         my $err = $@;
5744
5745         PVE::QemuConfig->lock_config($vmid, sub {
5746             $conf = PVE::QemuConfig->load_config($vmid);
5747             if ($err) {
5748                 # cleanup, but leave suspending lock, to indicate something went wrong
5749                 eval {
5750                     mon_cmd($vmid, "savevm-end");
5751                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5752                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5753                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5754                     PVE::QemuConfig->write_config($vmid, $conf);
5755                 };
5756                 warn $@ if $@;
5757                 die $err;
5758             }
5759
5760             die "lock changed unexpectedly\n"
5761                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5762
5763             mon_cmd($vmid, "quit");
5764             $conf->{lock} = 'suspended';
5765             PVE::QemuConfig->write_config($vmid, $conf);
5766         });
5767     }
5768 }
5769
5770 sub vm_resume {
5771     my ($vmid, $skiplock, $nocheck) = @_;
5772
5773     PVE::QemuConfig->lock_config($vmid, sub {
5774         my $res = mon_cmd($vmid, 'query-status');
5775         my $resume_cmd = 'cont';
5776         my $reset = 0;
5777
5778         if ($res->{status}) {
5779             return if $res->{status} eq 'running'; # job done, go home
5780             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5781             $reset = 1 if $res->{status} eq 'shutdown';
5782         }
5783
5784         if (!$nocheck) {
5785
5786             my $conf = PVE::QemuConfig->load_config($vmid);
5787
5788             PVE::QemuConfig->check_lock($conf)
5789                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5790         }
5791
5792         if ($reset) {
5793             # required if a VM shuts down during a backup and we get a resume
5794             # request before the backup finishes for example
5795             mon_cmd($vmid, "system_reset");
5796         }
5797         mon_cmd($vmid, $resume_cmd);
5798     });
5799 }
5800
5801 sub vm_sendkey {
5802     my ($vmid, $skiplock, $key) = @_;
5803
5804     PVE::QemuConfig->lock_config($vmid, sub {
5805
5806         my $conf = PVE::QemuConfig->load_config($vmid);
5807
5808         # there is no qmp command, so we use the human monitor command
5809         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5810         die $res if $res ne '';
5811     });
5812 }
5813
5814 # vzdump restore implementaion
5815
5816 sub tar_archive_read_firstfile {
5817     my $archive = shift;
5818
5819     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5820
5821     # try to detect archive type first
5822     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5823         die "unable to open file '$archive'\n";
5824     my $firstfile = <$fh>;
5825     kill 15, $pid;
5826     close $fh;
5827
5828     die "ERROR: archive contaions no data\n" if !$firstfile;
5829     chomp $firstfile;
5830
5831     return $firstfile;
5832 }
5833
5834 sub tar_restore_cleanup {
5835     my ($storecfg, $statfile) = @_;
5836
5837     print STDERR "starting cleanup\n";
5838
5839     if (my $fd = IO::File->new($statfile, "r")) {
5840         while (defined(my $line = <$fd>)) {
5841             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5842                 my $volid = $2;
5843                 eval {
5844                     if ($volid =~ m|^/|) {
5845                         unlink $volid || die 'unlink failed\n';
5846                     } else {
5847                         PVE::Storage::vdisk_free($storecfg, $volid);
5848                     }
5849                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5850                 };
5851                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5852             } else {
5853                 print STDERR "unable to parse line in statfile - $line";
5854             }
5855         }
5856         $fd->close();
5857     }
5858 }
5859
5860 sub restore_file_archive {
5861     my ($archive, $vmid, $user, $opts) = @_;
5862
5863     return restore_vma_archive($archive, $vmid, $user, $opts)
5864         if $archive eq '-';
5865
5866     my $info = PVE::Storage::archive_info($archive);
5867     my $format = $opts->{format} // $info->{format};
5868     my $comp = $info->{compression};
5869
5870     # try to detect archive format
5871     if ($format eq 'tar') {
5872         return restore_tar_archive($archive, $vmid, $user, $opts);
5873     } else {
5874         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5875     }
5876 }
5877
5878 # hepler to remove disks that will not be used after restore
5879 my $restore_cleanup_oldconf = sub {
5880     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5881
5882     PVE::QemuConfig->foreach_volume($oldconf, sub {
5883         my ($ds, $drive) = @_;
5884
5885         return if drive_is_cdrom($drive, 1);
5886
5887         my $volid = $drive->{file};
5888         return if !$volid || $volid =~ m|^/|;
5889
5890         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5891         return if !$path || !$owner || ($owner != $vmid);
5892
5893         # Note: only delete disk we want to restore
5894         # other volumes will become unused
5895         if ($virtdev_hash->{$ds}) {
5896             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5897             if (my $err = $@) {
5898                 warn $err;
5899             }
5900         }
5901     });
5902
5903     # delete vmstate files, after the restore we have no snapshots anymore
5904     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5905         my $snap = $oldconf->{snapshots}->{$snapname};
5906         if ($snap->{vmstate}) {
5907             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5908             if (my $err = $@) {
5909                 warn $err;
5910             }
5911         }
5912     }
5913 };
5914
5915 # Helper to parse vzdump backup device hints
5916 #
5917 # $rpcenv: Environment, used to ckeck storage permissions
5918 # $user: User ID, to check storage permissions
5919 # $storecfg: Storage configuration
5920 # $fh: the file handle for reading the configuration
5921 # $devinfo: should contain device sizes for all backu-up'ed devices
5922 # $options: backup options (pool, default storage)
5923 #
5924 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5925 my $parse_backup_hints = sub {
5926     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5927
5928     my $virtdev_hash = {};
5929
5930     while (defined(my $line = <$fh>)) {
5931         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5932             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5933             die "archive does not contain data for drive '$virtdev'\n"
5934                 if !$devinfo->{$devname};
5935
5936             if (defined($options->{storage})) {
5937                 $storeid = $options->{storage} || 'local';
5938             } elsif (!$storeid) {
5939                 $storeid = 'local';
5940             }
5941             $format = 'raw' if !$format;
5942             $devinfo->{$devname}->{devname} = $devname;
5943             $devinfo->{$devname}->{virtdev} = $virtdev;
5944             $devinfo->{$devname}->{format} = $format;
5945             $devinfo->{$devname}->{storeid} = $storeid;
5946
5947             # check permission on storage
5948             my $pool = $options->{pool}; # todo: do we need that?
5949             if ($user ne 'root@pam') {
5950                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5951             }
5952
5953             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5954         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5955             my $virtdev = $1;
5956             my $drive = parse_drive($virtdev, $2);
5957             if (drive_is_cloudinit($drive)) {
5958                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5959                 $storeid = $options->{storage} if defined ($options->{storage});
5960                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5961                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5962
5963                 $virtdev_hash->{$virtdev} = {
5964                     format => $format,
5965                     storeid => $storeid,
5966                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5967                     is_cloudinit => 1,
5968                 };
5969             }
5970         }
5971     }
5972
5973     return $virtdev_hash;
5974 };
5975
5976 # Helper to allocate and activate all volumes required for a restore
5977 #
5978 # $storecfg: Storage configuration
5979 # $virtdev_hash: as returned by parse_backup_hints()
5980 #
5981 # Returns: { $virtdev => $volid }
5982 my $restore_allocate_devices = sub {
5983     my ($storecfg, $virtdev_hash, $vmid) = @_;
5984
5985     my $map = {};
5986     foreach my $virtdev (sort keys %$virtdev_hash) {
5987         my $d = $virtdev_hash->{$virtdev};
5988         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5989         my $storeid = $d->{storeid};
5990         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5991
5992         # test if requested format is supported
5993         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5994         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5995         $d->{format} = $defFormat if !$supported;
5996
5997         my $name;
5998         if ($d->{is_cloudinit}) {
5999             $name = "vm-$vmid-cloudinit";
6000             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6001             if ($scfg->{path}) {
6002                 $name .= ".$d->{format}";
6003             }
6004         }
6005
6006         my $volid = PVE::Storage::vdisk_alloc(
6007             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6008
6009         print STDERR "new volume ID is '$volid'\n";
6010         $d->{volid} = $volid;
6011
6012         PVE::Storage::activate_volumes($storecfg, [$volid]);
6013
6014         $map->{$virtdev} = $volid;
6015     }
6016
6017     return $map;
6018 };
6019
6020 sub restore_update_config_line {
6021     my ($cookie, $map, $line, $unique) = @_;
6022
6023     return '' if $line =~ m/^\#qmdump\#/;
6024     return '' if $line =~ m/^\#vzdump\#/;
6025     return '' if $line =~ m/^lock:/;
6026     return '' if $line =~ m/^unused\d+:/;
6027     return '' if $line =~ m/^parent:/;
6028
6029     my $res = '';
6030
6031     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6032     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6033         # try to convert old 1.X settings
6034         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6035         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6036             my ($model, $macaddr) = split(/\=/, $devconfig);
6037             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6038             my $net = {
6039                 model => $model,
6040                 bridge => "vmbr$ind",
6041                 macaddr => $macaddr,
6042             };
6043             my $netstr = print_net($net);
6044
6045             $res .= "net$cookie->{netcount}: $netstr\n";
6046             $cookie->{netcount}++;
6047         }
6048     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6049         my ($id, $netstr) = ($1, $2);
6050         my $net = parse_net($netstr);
6051         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6052         $netstr = print_net($net);
6053         $res .= "$id: $netstr\n";
6054     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6055         my $virtdev = $1;
6056         my $value = $3;
6057         my $di = parse_drive($virtdev, $value);
6058         if (defined($di->{backup}) && !$di->{backup}) {
6059             $res .= "#$line";
6060         } elsif ($map->{$virtdev}) {
6061             delete $di->{format}; # format can change on restore
6062             $di->{file} = $map->{$virtdev};
6063             $value = print_drive($di);
6064             $res .= "$virtdev: $value\n";
6065         } else {
6066             $res .= $line;
6067         }
6068     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6069         my $vmgenid = $1;
6070         if ($vmgenid ne '0') {
6071             # always generate a new vmgenid if there was a valid one setup
6072             $vmgenid = generate_uuid();
6073         }
6074         $res .= "vmgenid: $vmgenid\n";
6075     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6076         my ($uuid, $uuid_str);
6077         UUID::generate($uuid);
6078         UUID::unparse($uuid, $uuid_str);
6079         my $smbios1 = parse_smbios1($2);
6080         $smbios1->{uuid} = $uuid_str;
6081         $res .= $1.print_smbios1($smbios1)."\n";
6082     } else {
6083         $res .= $line;
6084     }
6085
6086     return $res;
6087 }
6088
6089 my $restore_deactivate_volumes = sub {
6090     my ($storecfg, $devinfo) = @_;
6091
6092     my $vollist = [];
6093     foreach my $devname (keys %$devinfo) {
6094         my $volid = $devinfo->{$devname}->{volid};
6095         push @$vollist, $volid if $volid;
6096     }
6097
6098     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6099 };
6100
6101 my $restore_destroy_volumes = sub {
6102     my ($storecfg, $devinfo) = @_;
6103
6104     foreach my $devname (keys %$devinfo) {
6105         my $volid = $devinfo->{$devname}->{volid};
6106         next if !$volid;
6107         eval {
6108             if ($volid =~ m|^/|) {
6109                 unlink $volid || die 'unlink failed\n';
6110             } else {
6111                 PVE::Storage::vdisk_free($storecfg, $volid);
6112             }
6113             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6114         };
6115         print STDERR "unable to cleanup '$volid' - $@" if $@;
6116     }
6117 };
6118
6119 sub scan_volids {
6120     my ($cfg, $vmid) = @_;
6121
6122     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6123
6124     my $volid_hash = {};
6125     foreach my $storeid (keys %$info) {
6126         foreach my $item (@{$info->{$storeid}}) {
6127             next if !($item->{volid} && $item->{size});
6128             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6129             $volid_hash->{$item->{volid}} = $item;
6130         }
6131     }
6132
6133     return $volid_hash;
6134 }
6135
6136 sub update_disk_config {
6137     my ($vmid, $conf, $volid_hash) = @_;
6138
6139     my $changes;
6140     my $prefix = "VM $vmid";
6141
6142     # used and unused disks
6143     my $referenced = {};
6144
6145     # Note: it is allowed to define multiple storages with same path (alias), so
6146     # we need to check both 'volid' and real 'path' (two different volid can point
6147     # to the same path).
6148
6149     my $referencedpath = {};
6150
6151     # update size info
6152     PVE::QemuConfig->foreach_volume($conf, sub {
6153         my ($opt, $drive) = @_;
6154
6155         my $volid = $drive->{file};
6156         return if !$volid;
6157         my $volume = $volid_hash->{$volid};
6158
6159         # mark volid as "in-use" for next step
6160         $referenced->{$volid} = 1;
6161         if ($volume && (my $path = $volume->{path})) {
6162             $referencedpath->{$path} = 1;
6163         }
6164
6165         return if drive_is_cdrom($drive);
6166         return if !$volume;
6167
6168         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6169         if (defined($updated)) {
6170             $changes = 1;
6171             $conf->{$opt} = print_drive($updated);
6172             print "$prefix ($opt): $msg\n";
6173         }
6174     });
6175
6176     # remove 'unusedX' entry if volume is used
6177     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6178         my ($opt, $drive) = @_;
6179
6180         my $volid = $drive->{file};
6181         return if !$volid;
6182
6183         my $path;
6184         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6185         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6186             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6187             $changes = 1;
6188             delete $conf->{$opt};
6189         }
6190
6191         $referenced->{$volid} = 1;
6192         $referencedpath->{$path} = 1 if $path;
6193     });
6194
6195     foreach my $volid (sort keys %$volid_hash) {
6196         next if $volid =~ m/vm-$vmid-state-/;
6197         next if $referenced->{$volid};
6198         my $path = $volid_hash->{$volid}->{path};
6199         next if !$path; # just to be sure
6200         next if $referencedpath->{$path};
6201         $changes = 1;
6202         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6203         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6204         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6205     }
6206
6207     return $changes;
6208 }
6209
6210 sub rescan {
6211     my ($vmid, $nolock, $dryrun) = @_;
6212
6213     my $cfg = PVE::Storage::config();
6214
6215     print "rescan volumes...\n";
6216     my $volid_hash = scan_volids($cfg, $vmid);
6217
6218     my $updatefn =  sub {
6219         my ($vmid) = @_;
6220
6221         my $conf = PVE::QemuConfig->load_config($vmid);
6222
6223         PVE::QemuConfig->check_lock($conf);
6224
6225         my $vm_volids = {};
6226         foreach my $volid (keys %$volid_hash) {
6227             my $info = $volid_hash->{$volid};
6228             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6229         }
6230
6231         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6232
6233         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6234     };
6235
6236     if (defined($vmid)) {
6237         if ($nolock) {
6238             &$updatefn($vmid);
6239         } else {
6240             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6241         }
6242     } else {
6243         my $vmlist = config_list();
6244         foreach my $vmid (keys %$vmlist) {
6245             if ($nolock) {
6246                 &$updatefn($vmid);
6247             } else {
6248                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6249             }
6250         }
6251     }
6252 }
6253
6254 sub restore_proxmox_backup_archive {
6255     my ($archive, $vmid, $user, $options) = @_;
6256
6257     my $storecfg = PVE::Storage::config();
6258
6259     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6260     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6261
6262     my $fingerprint = $scfg->{fingerprint};
6263     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6264
6265     my $repo = PVE::PBSClient::get_repository($scfg);
6266
6267     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6268     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6269     local $ENV{PBS_PASSWORD} = $password;
6270     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6271
6272     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6273         PVE::Storage::parse_volname($storecfg, $archive);
6274
6275     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6276
6277     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6278
6279     my $tmpdir = "/var/tmp/vzdumptmp$$";
6280     rmtree $tmpdir;
6281     mkpath $tmpdir;
6282
6283     my $conffile = PVE::QemuConfig->config_file($vmid);
6284      # disable interrupts (always do cleanups)
6285     local $SIG{INT} =
6286         local $SIG{TERM} =
6287         local $SIG{QUIT} =
6288         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6289
6290     # Note: $oldconf is undef if VM does not exists
6291     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6292     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6293     my $new_conf_raw = '';
6294
6295     my $rpcenv = PVE::RPCEnvironment::get();
6296     my $devinfo = {};
6297
6298     eval {
6299         # enable interrupts
6300         local $SIG{INT} =
6301             local $SIG{TERM} =
6302             local $SIG{QUIT} =
6303             local $SIG{HUP} =
6304             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6305
6306         my $cfgfn = "$tmpdir/qemu-server.conf";
6307         my $firewall_config_fn = "$tmpdir/fw.conf";
6308         my $index_fn = "$tmpdir/index.json";
6309
6310         my $cmd = "restore";
6311
6312         my $param = [$pbs_backup_name, "index.json", $index_fn];
6313         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6314         my $index = PVE::Tools::file_get_contents($index_fn);
6315         $index = decode_json($index);
6316
6317         # print Dumper($index);
6318         foreach my $info (@{$index->{files}}) {
6319             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6320                 my $devname = $1;
6321                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6322                     $devinfo->{$devname}->{size} = $1;
6323                 } else {
6324                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6325                 }
6326             }
6327         }
6328
6329         my $is_qemu_server_backup = scalar(
6330             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6331         );
6332         if (!$is_qemu_server_backup) {
6333             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6334         }
6335         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6336
6337         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6338         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6339
6340         if ($has_firewall_config) {
6341             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6342             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6343
6344             my $pve_firewall_dir = '/etc/pve/firewall';
6345             mkdir $pve_firewall_dir; # make sure the dir exists
6346             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6347         }
6348
6349         my $fh = IO::File->new($cfgfn, "r") ||
6350             die "unable to read qemu-server.conf - $!\n";
6351
6352         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6353
6354         # fixme: rate limit?
6355
6356         # create empty/temp config
6357         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6358
6359         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6360
6361         # allocate volumes
6362         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6363
6364         if (!$options->{live}) {
6365             foreach my $virtdev (sort keys %$virtdev_hash) {
6366                 my $d = $virtdev_hash->{$virtdev};
6367                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6368
6369                 my $volid = $d->{volid};
6370
6371                 my $path = PVE::Storage::path($storecfg, $volid);
6372
6373                 my $pbs_restore_cmd = [
6374                     '/usr/bin/pbs-restore',
6375                     '--repository', $repo,
6376                     $pbs_backup_name,
6377                     "$d->{devname}.img.fidx",
6378                     $path,
6379                     '--verbose',
6380                     ];
6381
6382                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6383                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6384
6385                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6386                     push @$pbs_restore_cmd, '--skip-zero';
6387                 }
6388
6389                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6390                 print "restore proxmox backup image: $dbg_cmdstring\n";
6391                 run_command($pbs_restore_cmd);
6392             }
6393         }
6394
6395         $fh->seek(0, 0) || die "seek failed - $!\n";
6396
6397         my $cookie = { netcount => 0 };
6398         while (defined(my $line = <$fh>)) {
6399             $new_conf_raw .= restore_update_config_line(
6400                 $cookie,
6401                 $map,
6402                 $line,
6403                 $options->{unique},
6404             );
6405         }
6406
6407         $fh->close();
6408     };
6409     my $err = $@;
6410
6411     if ($err || !$options->{live}) {
6412         $restore_deactivate_volumes->($storecfg, $devinfo);
6413     }
6414
6415     rmtree $tmpdir;
6416
6417     if ($err) {
6418         $restore_destroy_volumes->($storecfg, $devinfo);
6419         die $err;
6420     }
6421
6422     if ($options->{live}) {
6423         # keep lock during live-restore
6424         $new_conf_raw .= "\nlock: create";
6425     }
6426
6427     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6428
6429     PVE::Cluster::cfs_update(); # make sure we read new file
6430
6431     eval { rescan($vmid, 1); };
6432     warn $@ if $@;
6433
6434     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6435
6436     if ($options->{live}) {
6437         # enable interrupts
6438         local $SIG{INT} =
6439             local $SIG{TERM} =
6440             local $SIG{QUIT} =
6441             local $SIG{HUP} =
6442             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6443
6444         my $conf = PVE::QemuConfig->load_config($vmid);
6445         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6446
6447         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6448
6449         PVE::QemuConfig->remove_lock($vmid, "create");
6450     }
6451 }
6452
6453 sub pbs_live_restore {
6454     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6455
6456     print "starting VM for live-restore\n";
6457     print "repository: '$repo', snapshot: '$snap'\n";
6458
6459     my $pbs_backing = {};
6460     for my $ds (keys %$restored_disks) {
6461         $ds =~ m/^drive-(.*)$/;
6462         my $confname = $1;
6463         $pbs_backing->{$confname} = {
6464             repository => $repo,
6465             snapshot => $snap,
6466             archive => "$ds.img.fidx",
6467         };
6468         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6469
6470         my $drive = parse_drive($confname, $conf->{$confname});
6471         print "restoring '$ds' to '$drive->{file}'\n";
6472     }
6473
6474     my $drives_streamed = 0;
6475     eval {
6476         # make sure HA doesn't interrupt our restore by stopping the VM
6477         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6478             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6479         }
6480
6481         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6482         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6483         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6484
6485         my $qmeventd_fd = register_qmeventd_handle($vmid);
6486
6487         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6488         # this will effectively collapse the backing image chain consisting of
6489         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6490         # removes itself once all backing images vanish with 'auto-remove=on')
6491         my $jobs = {};
6492         for my $ds (sort keys %$restored_disks) {
6493             my $job_id = "restore-$ds";
6494             mon_cmd($vmid, 'block-stream',
6495                 'job-id' => $job_id,
6496                 device => "$ds",
6497             );
6498             $jobs->{$job_id} = {};
6499         }
6500
6501         mon_cmd($vmid, 'cont');
6502         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6503
6504         print "restore-drive jobs finished successfully, removing all tracking block devices"
6505             ." to disconnect from Proxmox Backup Server\n";
6506
6507         for my $ds (sort keys %$restored_disks) {
6508             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6509         }
6510
6511         close($qmeventd_fd);
6512     };
6513
6514     my $err = $@;
6515
6516     if ($err) {
6517         warn "An error occured during live-restore: $err\n";
6518         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6519         die "live-restore failed\n";
6520     }
6521 }
6522
6523 sub restore_vma_archive {
6524     my ($archive, $vmid, $user, $opts, $comp) = @_;
6525
6526     my $readfrom = $archive;
6527
6528     my $cfg = PVE::Storage::config();
6529     my $commands = [];
6530     my $bwlimit = $opts->{bwlimit};
6531
6532     my $dbg_cmdstring = '';
6533     my $add_pipe = sub {
6534         my ($cmd) = @_;
6535         push @$commands, $cmd;
6536         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6537         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6538         $readfrom = '-';
6539     };
6540
6541     my $input = undef;
6542     if ($archive eq '-') {
6543         $input = '<&STDIN';
6544     } else {
6545         # If we use a backup from a PVE defined storage we also consider that
6546         # storage's rate limit:
6547         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6548         if (defined($volid)) {
6549             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6550             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6551             if ($readlimit) {
6552                 print STDERR "applying read rate limit: $readlimit\n";
6553                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6554                 $add_pipe->($cstream);
6555             }
6556         }
6557     }
6558
6559     if ($comp) {
6560         my $info = PVE::Storage::decompressor_info('vma', $comp);
6561         my $cmd = $info->{decompressor};
6562         push @$cmd, $readfrom;
6563         $add_pipe->($cmd);
6564     }
6565
6566     my $tmpdir = "/var/tmp/vzdumptmp$$";
6567     rmtree $tmpdir;
6568
6569     # disable interrupts (always do cleanups)
6570     local $SIG{INT} =
6571         local $SIG{TERM} =
6572         local $SIG{QUIT} =
6573         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6574
6575     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6576     POSIX::mkfifo($mapfifo, 0600);
6577     my $fifofh;
6578     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6579
6580     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6581
6582     my $oldtimeout;
6583     my $timeout = 5;
6584
6585     my $devinfo = {};
6586
6587     my $rpcenv = PVE::RPCEnvironment::get();
6588
6589     my $conffile = PVE::QemuConfig->config_file($vmid);
6590
6591     # Note: $oldconf is undef if VM does not exist
6592     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6593     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6594     my $new_conf_raw = '';
6595
6596     my %storage_limits;
6597
6598     my $print_devmap = sub {
6599         my $cfgfn = "$tmpdir/qemu-server.conf";
6600
6601         # we can read the config - that is already extracted
6602         my $fh = IO::File->new($cfgfn, "r") ||
6603             die "unable to read qemu-server.conf - $!\n";
6604
6605         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6606         if (-f $fwcfgfn) {
6607             my $pve_firewall_dir = '/etc/pve/firewall';
6608             mkdir $pve_firewall_dir; # make sure the dir exists
6609             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6610         }
6611
6612         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6613
6614         foreach my $info (values %{$virtdev_hash}) {
6615             my $storeid = $info->{storeid};
6616             next if defined($storage_limits{$storeid});
6617
6618             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6619             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6620             $storage_limits{$storeid} = $limit * 1024;
6621         }
6622
6623         foreach my $devname (keys %$devinfo) {
6624             die "found no device mapping information for device '$devname'\n"
6625                 if !$devinfo->{$devname}->{virtdev};
6626         }
6627
6628         # create empty/temp config
6629         if ($oldconf) {
6630             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6631             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6632         }
6633
6634         # allocate volumes
6635         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6636
6637         # print restore information to $fifofh
6638         foreach my $virtdev (sort keys %$virtdev_hash) {
6639             my $d = $virtdev_hash->{$virtdev};
6640             next if $d->{is_cloudinit}; # no need to restore cloudinit
6641
6642             my $storeid = $d->{storeid};
6643             my $volid = $d->{volid};
6644
6645             my $map_opts = '';
6646             if (my $limit = $storage_limits{$storeid}) {
6647                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6648             }
6649
6650             my $write_zeros = 1;
6651             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6652                 $write_zeros = 0;
6653             }
6654
6655             my $path = PVE::Storage::path($cfg, $volid);
6656
6657             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6658
6659             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6660         }
6661
6662         $fh->seek(0, 0) || die "seek failed - $!\n";
6663
6664         my $cookie = { netcount => 0 };
6665         while (defined(my $line = <$fh>)) {
6666             $new_conf_raw .= restore_update_config_line(
6667                 $cookie,
6668                 $map,
6669                 $line,
6670                 $opts->{unique},
6671             );
6672         }
6673
6674         $fh->close();
6675     };
6676
6677     eval {
6678         # enable interrupts
6679         local $SIG{INT} =
6680             local $SIG{TERM} =
6681             local $SIG{QUIT} =
6682             local $SIG{HUP} =
6683             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6684         local $SIG{ALRM} = sub { die "got timeout\n"; };
6685
6686         $oldtimeout = alarm($timeout);
6687
6688         my $parser = sub {
6689             my $line = shift;
6690
6691             print "$line\n";
6692
6693             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6694                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6695                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6696             } elsif ($line =~ m/^CTIME: /) {
6697                 # we correctly received the vma config, so we can disable
6698                 # the timeout now for disk allocation (set to 10 minutes, so
6699                 # that we always timeout if something goes wrong)
6700                 alarm(600);
6701                 &$print_devmap();
6702                 print $fifofh "done\n";
6703                 my $tmp = $oldtimeout || 0;
6704                 $oldtimeout = undef;
6705                 alarm($tmp);
6706                 close($fifofh);
6707                 $fifofh = undef;
6708             }
6709         };
6710
6711         print "restore vma archive: $dbg_cmdstring\n";
6712         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6713     };
6714     my $err = $@;
6715
6716     alarm($oldtimeout) if $oldtimeout;
6717
6718     $restore_deactivate_volumes->($cfg, $devinfo);
6719
6720     close($fifofh) if $fifofh;
6721     unlink $mapfifo;
6722     rmtree $tmpdir;
6723
6724     if ($err) {
6725         $restore_destroy_volumes->($cfg, $devinfo);
6726         die $err;
6727     }
6728
6729     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6730
6731     PVE::Cluster::cfs_update(); # make sure we read new file
6732
6733     eval { rescan($vmid, 1); };
6734     warn $@ if $@;
6735
6736     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6737 }
6738
6739 sub restore_tar_archive {
6740     my ($archive, $vmid, $user, $opts) = @_;
6741
6742     if ($archive ne '-') {
6743         my $firstfile = tar_archive_read_firstfile($archive);
6744         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6745             if $firstfile ne 'qemu-server.conf';
6746     }
6747
6748     my $storecfg = PVE::Storage::config();
6749
6750     # avoid zombie disks when restoring over an existing VM -> cleanup first
6751     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6752     # skiplock=1 because qmrestore has set the 'create' lock itself already
6753     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6754     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6755
6756     my $tocmd = "/usr/lib/qemu-server/qmextract";
6757
6758     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6759     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6760     $tocmd .= ' --prealloc' if $opts->{prealloc};
6761     $tocmd .= ' --info' if $opts->{info};
6762
6763     # tar option "xf" does not autodetect compression when read from STDIN,
6764     # so we pipe to zcat
6765     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6766         PVE::Tools::shellquote("--to-command=$tocmd");
6767
6768     my $tmpdir = "/var/tmp/vzdumptmp$$";
6769     mkpath $tmpdir;
6770
6771     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6772     local $ENV{VZDUMP_VMID} = $vmid;
6773     local $ENV{VZDUMP_USER} = $user;
6774
6775     my $conffile = PVE::QemuConfig->config_file($vmid);
6776     my $new_conf_raw = '';
6777
6778     # disable interrupts (always do cleanups)
6779     local $SIG{INT} =
6780         local $SIG{TERM} =
6781         local $SIG{QUIT} =
6782         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6783
6784     eval {
6785         # enable interrupts
6786         local $SIG{INT} =
6787             local $SIG{TERM} =
6788             local $SIG{QUIT} =
6789             local $SIG{HUP} =
6790             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6791
6792         if ($archive eq '-') {
6793             print "extracting archive from STDIN\n";
6794             run_command($cmd, input => "<&STDIN");
6795         } else {
6796             print "extracting archive '$archive'\n";
6797             run_command($cmd);
6798         }
6799
6800         return if $opts->{info};
6801
6802         # read new mapping
6803         my $map = {};
6804         my $statfile = "$tmpdir/qmrestore.stat";
6805         if (my $fd = IO::File->new($statfile, "r")) {
6806             while (defined (my $line = <$fd>)) {
6807                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6808                     $map->{$1} = $2 if $1;
6809                 } else {
6810                     print STDERR "unable to parse line in statfile - $line\n";
6811                 }
6812             }
6813             $fd->close();
6814         }
6815
6816         my $confsrc = "$tmpdir/qemu-server.conf";
6817
6818         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6819
6820         my $cookie = { netcount => 0 };
6821         while (defined (my $line = <$srcfd>)) {
6822             $new_conf_raw .= restore_update_config_line(
6823                 $cookie,
6824                 $map,
6825                 $line,
6826                 $opts->{unique},
6827             );
6828         }
6829
6830         $srcfd->close();
6831     };
6832     if (my $err = $@) {
6833         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6834         die $err;
6835     }
6836
6837     rmtree $tmpdir;
6838
6839     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6840
6841     PVE::Cluster::cfs_update(); # make sure we read new file
6842
6843     eval { rescan($vmid, 1); };
6844     warn $@ if $@;
6845 };
6846
6847 sub foreach_storage_used_by_vm {
6848     my ($conf, $func) = @_;
6849
6850     my $sidhash = {};
6851
6852     PVE::QemuConfig->foreach_volume($conf, sub {
6853         my ($ds, $drive) = @_;
6854         return if drive_is_cdrom($drive);
6855
6856         my $volid = $drive->{file};
6857
6858         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6859         $sidhash->{$sid} = $sid if $sid;
6860     });
6861
6862     foreach my $sid (sort keys %$sidhash) {
6863         &$func($sid);
6864     }
6865 }
6866
6867 my $qemu_snap_storage = {
6868     rbd => 1,
6869 };
6870 sub do_snapshots_with_qemu {
6871     my ($storecfg, $volid) = @_;
6872
6873     my $storage_name = PVE::Storage::parse_volume_id($volid);
6874     my $scfg = $storecfg->{ids}->{$storage_name};
6875     die "could not find storage '$storage_name'\n" if !defined($scfg);
6876
6877     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6878         return 1;
6879     }
6880
6881     if ($volid =~ m/\.(qcow2|qed)$/){
6882         return 1;
6883     }
6884
6885     return;
6886 }
6887
6888 sub qga_check_running {
6889     my ($vmid, $nowarn) = @_;
6890
6891     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6892     if ($@) {
6893         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6894         return 0;
6895     }
6896     return 1;
6897 }
6898
6899 sub template_create {
6900     my ($vmid, $conf, $disk) = @_;
6901
6902     my $storecfg = PVE::Storage::config();
6903
6904     PVE::QemuConfig->foreach_volume($conf, sub {
6905         my ($ds, $drive) = @_;
6906
6907         return if drive_is_cdrom($drive);
6908         return if $disk && $ds ne $disk;
6909
6910         my $volid = $drive->{file};
6911         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6912
6913         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6914         $drive->{file} = $voliddst;
6915         $conf->{$ds} = print_drive($drive);
6916         PVE::QemuConfig->write_config($vmid, $conf);
6917     });
6918 }
6919
6920 sub convert_iscsi_path {
6921     my ($path) = @_;
6922
6923     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6924         my $portal = $1;
6925         my $target = $2;
6926         my $lun = $3;
6927
6928         my $initiator_name = get_initiator_name();
6929
6930         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6931                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6932     }
6933
6934     die "cannot convert iscsi path '$path', unkown format\n";
6935 }
6936
6937 sub qemu_img_convert {
6938     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6939
6940     my $storecfg = PVE::Storage::config();
6941     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6942     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6943
6944     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6945
6946     my $cachemode;
6947     my $src_path;
6948     my $src_is_iscsi = 0;
6949     my $src_format;
6950
6951     if ($src_storeid) {
6952         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6953         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6954         $src_format = qemu_img_format($src_scfg, $src_volname);
6955         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6956         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6957         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6958     } elsif (-f $src_volid) {
6959         $src_path = $src_volid;
6960         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6961             $src_format = $1;
6962         }
6963     }
6964
6965     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6966
6967     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6968     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6969     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6970     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6971
6972     my $cmd = [];
6973     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6974     push @$cmd, '-l', "snapshot.name=$snapname"
6975         if $snapname && $src_format && $src_format eq "qcow2";
6976     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6977     push @$cmd, '-T', $cachemode if defined($cachemode);
6978
6979     if ($src_is_iscsi) {
6980         push @$cmd, '--image-opts';
6981         $src_path = convert_iscsi_path($src_path);
6982     } elsif ($src_format) {
6983         push @$cmd, '-f', $src_format;
6984     }
6985
6986     if ($dst_is_iscsi) {
6987         push @$cmd, '--target-image-opts';
6988         $dst_path = convert_iscsi_path($dst_path);
6989     } else {
6990         push @$cmd, '-O', $dst_format;
6991     }
6992
6993     push @$cmd, $src_path;
6994
6995     if (!$dst_is_iscsi && $is_zero_initialized) {
6996         push @$cmd, "zeroinit:$dst_path";
6997     } else {
6998         push @$cmd, $dst_path;
6999     }
7000
7001     my $parser = sub {
7002         my $line = shift;
7003         if($line =~ m/\((\S+)\/100\%\)/){
7004             my $percent = $1;
7005             my $transferred = int($size * $percent / 100);
7006             my $total_h = render_bytes($size, 1);
7007             my $transferred_h = render_bytes($transferred, 1);
7008
7009             print "transferred $transferred_h of $total_h ($percent%)\n";
7010         }
7011
7012     };
7013
7014     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7015     my $err = $@;
7016     die "copy failed: $err" if $err;
7017 }
7018
7019 sub qemu_img_format {
7020     my ($scfg, $volname) = @_;
7021
7022     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7023         return $1;
7024     } else {
7025         return "raw";
7026     }
7027 }
7028
7029 sub qemu_drive_mirror {
7030     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7031
7032     $jobs = {} if !$jobs;
7033
7034     my $qemu_target;
7035     my $format;
7036     $jobs->{"drive-$drive"} = {};
7037
7038     if ($dst_volid =~ /^nbd:/) {
7039         $qemu_target = $dst_volid;
7040         $format = "nbd";
7041     } else {
7042         my $storecfg = PVE::Storage::config();
7043         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7044
7045         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7046
7047         $format = qemu_img_format($dst_scfg, $dst_volname);
7048
7049         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7050
7051         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7052     }
7053
7054     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7055     $opts->{format} = $format if $format;
7056
7057     if (defined($src_bitmap)) {
7058         $opts->{sync} = 'incremental';
7059         $opts->{bitmap} = $src_bitmap;
7060         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7061     }
7062
7063     if (defined($bwlimit)) {
7064         $opts->{speed} = $bwlimit * 1024;
7065         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7066     } else {
7067         print "drive mirror is starting for drive-$drive\n";
7068     }
7069
7070     # if a job already runs for this device we get an error, catch it for cleanup
7071     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7072     if (my $err = $@) {
7073         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7074         warn "$@\n" if $@;
7075         die "mirroring error: $err\n";
7076     }
7077
7078     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7079 }
7080
7081 # $completion can be either
7082 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7083 # 'cancel': wait until all jobs are ready, block-job-cancel them
7084 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7085 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7086 sub qemu_drive_mirror_monitor {
7087     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7088
7089     $completion //= 'complete';
7090     $op //= "mirror";
7091
7092     eval {
7093         my $err_complete = 0;
7094
7095         my $starttime = time ();
7096         while (1) {
7097             die "block job ('$op') timed out\n" if $err_complete > 300;
7098
7099             my $stats = mon_cmd($vmid, "query-block-jobs");
7100             my $ctime = time();
7101
7102             my $running_jobs = {};
7103             for my $stat (@$stats) {
7104                 next if $stat->{type} ne $op;
7105                 $running_jobs->{$stat->{device}} = $stat;
7106             }
7107
7108             my $readycounter = 0;
7109
7110             for my $job_id (sort keys %$jobs) {
7111                 my $job = $running_jobs->{$job_id};
7112
7113                 my $vanished = !defined($job);
7114                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7115                 if($complete || ($vanished && $completion eq 'auto')) {
7116                     print "$job_id: $op-job finished\n";
7117                     delete $jobs->{$job_id};
7118                     next;
7119                 }
7120
7121                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7122
7123                 my $busy = $job->{busy};
7124                 my $ready = $job->{ready};
7125                 if (my $total = $job->{len}) {
7126                     my $transferred = $job->{offset} || 0;
7127                     my $remaining = $total - $transferred;
7128                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7129
7130                     my $duration = $ctime - $starttime;
7131                     my $total_h = render_bytes($total, 1);
7132                     my $transferred_h = render_bytes($transferred, 1);
7133
7134                     my $status = sprintf(
7135                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7136                         render_duration($duration),
7137                     );
7138
7139                     if ($ready) {
7140                         if ($busy) {
7141                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7142                         } else {
7143                             $status .= ", ready";
7144                         }
7145                     }
7146                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7147                     $jobs->{$job_id}->{ready} = $ready;
7148                 }
7149
7150                 $readycounter++ if $job->{ready};
7151             }
7152
7153             last if scalar(keys %$jobs) == 0;
7154
7155             if ($readycounter == scalar(keys %$jobs)) {
7156                 print "all '$op' jobs are ready\n";
7157
7158                 # do the complete later (or has already been done)
7159                 last if $completion eq 'skip' || $completion eq 'auto';
7160
7161                 if ($vmiddst && $vmiddst != $vmid) {
7162                     my $agent_running = $qga && qga_check_running($vmid);
7163                     if ($agent_running) {
7164                         print "freeze filesystem\n";
7165                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7166                     } else {
7167                         print "suspend vm\n";
7168                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7169                     }
7170
7171                     # if we clone a disk for a new target vm, we don't switch the disk
7172                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7173
7174                     if ($agent_running) {
7175                         print "unfreeze filesystem\n";
7176                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7177                     } else {
7178                         print "resume vm\n";
7179                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7180                     }
7181
7182                     last;
7183                 } else {
7184
7185                     for my $job_id (sort keys %$jobs) {
7186                         # try to switch the disk if source and destination are on the same guest
7187                         print "$job_id: Completing block job_id...\n";
7188
7189                         my $op;
7190                         if ($completion eq 'complete') {
7191                             $op = 'block-job-complete';
7192                         } elsif ($completion eq 'cancel') {
7193                             $op = 'block-job-cancel';
7194                         } else {
7195                             die "invalid completion value: $completion\n";
7196                         }
7197                         eval { mon_cmd($vmid, $op, device => $job_id) };
7198                         if ($@ =~ m/cannot be completed/) {
7199                             print "$job_id: block job cannot be completed, trying again.\n";
7200                             $err_complete++;
7201                         }else {
7202                             print "$job_id: Completed successfully.\n";
7203                             $jobs->{$job_id}->{complete} = 1;
7204                         }
7205                     }
7206                 }
7207             }
7208             sleep 1;
7209         }
7210     };
7211     my $err = $@;
7212
7213     if ($err) {
7214         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7215         die "block job ($op) error: $err";
7216     }
7217 }
7218
7219 sub qemu_blockjobs_cancel {
7220     my ($vmid, $jobs) = @_;
7221
7222     foreach my $job (keys %$jobs) {
7223         print "$job: Cancelling block job\n";
7224         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7225         $jobs->{$job}->{cancel} = 1;
7226     }
7227
7228     while (1) {
7229         my $stats = mon_cmd($vmid, "query-block-jobs");
7230
7231         my $running_jobs = {};
7232         foreach my $stat (@$stats) {
7233             $running_jobs->{$stat->{device}} = $stat;
7234         }
7235
7236         foreach my $job (keys %$jobs) {
7237
7238             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7239                 print "$job: Done.\n";
7240                 delete $jobs->{$job};
7241             }
7242         }
7243
7244         last if scalar(keys %$jobs) == 0;
7245
7246         sleep 1;
7247     }
7248 }
7249
7250 sub clone_disk {
7251     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7252         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7253
7254     my $newvolid;
7255
7256     if (!$full) {
7257         print "create linked clone of drive $drivename ($drive->{file})\n";
7258         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7259         push @$newvollist, $newvolid;
7260     } else {
7261
7262         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7263         $storeid = $storage if $storage;
7264
7265         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7266
7267         print "create full clone of drive $drivename ($drive->{file})\n";
7268         my $name = undef;
7269         my $size = undef;
7270         if (drive_is_cloudinit($drive)) {
7271             $name = "vm-$newvmid-cloudinit";
7272             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7273             if ($scfg->{path}) {
7274                 $name .= ".$dst_format";
7275             }
7276             $snapname = undef;
7277             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7278         } elsif ($drivename eq 'efidisk0') {
7279             $size = get_efivars_size($conf);
7280         } else {
7281             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7282         }
7283         $newvolid = PVE::Storage::vdisk_alloc(
7284             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7285         );
7286         push @$newvollist, $newvolid;
7287
7288         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7289
7290         if (drive_is_cloudinit($drive)) {
7291             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7292             # if this is the case, we have to complete any block-jobs still there from
7293             # previous drive-mirrors
7294             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7295                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7296             }
7297             goto no_data_clone;
7298         }
7299
7300         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7301         if (!$running || $snapname) {
7302             # TODO: handle bwlimits
7303             if ($drivename eq 'efidisk0') {
7304                 # the relevant data on the efidisk may be smaller than the source
7305                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7306                 # that is given by the OVMF_VARS.fd
7307                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7308                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7309
7310                 # better for Ceph if block size is not too small, see bug #3324
7311                 my $bs = 1024*1024;
7312
7313                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7314                     "if=$src_path", "of=$dst_path"]);
7315             } else {
7316                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7317             }
7318         } else {
7319
7320             my $kvmver = get_running_qemu_version ($vmid);
7321             if (!min_version($kvmver, 2, 7)) {
7322                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7323                     if $drive->{iothread};
7324             }
7325
7326             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7327                 $completion, $qga, $bwlimit);
7328         }
7329     }
7330
7331 no_data_clone:
7332     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7333
7334     my $disk = $drive;
7335     $disk->{format} = undef;
7336     $disk->{file} = $newvolid;
7337     $disk->{size} = $size if defined($size);
7338
7339     return $disk;
7340 }
7341
7342 sub get_running_qemu_version {
7343     my ($vmid) = @_;
7344     my $res = mon_cmd($vmid, "query-version");
7345     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7346 }
7347
7348 sub qemu_use_old_bios_files {
7349     my ($machine_type) = @_;
7350
7351     return if !$machine_type;
7352
7353     my $use_old_bios_files = undef;
7354
7355     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7356         $machine_type = $1;
7357         $use_old_bios_files = 1;
7358     } else {
7359         my $version = extract_version($machine_type, kvm_user_version());
7360         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7361         # load new efi bios files on migration. So this hack is required to allow
7362         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7363         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7364         $use_old_bios_files = !min_version($version, 2, 4);
7365     }
7366
7367     return ($use_old_bios_files, $machine_type);
7368 }
7369
7370 sub get_efivars_size {
7371     my ($conf) = @_;
7372     my $arch = get_vm_arch($conf);
7373     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7374     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7375     return -s $ovmf_vars;
7376 }
7377
7378 sub update_efidisk_size {
7379     my ($conf) = @_;
7380
7381     return if !defined($conf->{efidisk0});
7382
7383     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7384     $disk->{size} = get_efivars_size($conf);
7385     $conf->{efidisk0} = print_drive($disk);
7386
7387     return;
7388 }
7389
7390 sub create_efidisk($$$$$) {
7391     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7392
7393     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7394     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7395
7396     my $vars_size_b = -s $ovmf_vars;
7397     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7398     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7399     PVE::Storage::activate_volumes($storecfg, [$volid]);
7400
7401     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7402     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7403
7404     return ($volid, $size/1024);
7405 }
7406
7407 sub vm_iothreads_list {
7408     my ($vmid) = @_;
7409
7410     my $res = mon_cmd($vmid, 'query-iothreads');
7411
7412     my $iothreads = {};
7413     foreach my $iothread (@$res) {
7414         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7415     }
7416
7417     return $iothreads;
7418 }
7419
7420 sub scsihw_infos {
7421     my ($conf, $drive) = @_;
7422
7423     my $maxdev = 0;
7424
7425     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7426         $maxdev = 7;
7427     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7428         $maxdev = 1;
7429     } else {
7430         $maxdev = 256;
7431     }
7432
7433     my $controller = int($drive->{index} / $maxdev);
7434     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7435         ? "virtioscsi"
7436         : "scsihw";
7437
7438     return ($maxdev, $controller, $controller_prefix);
7439 }
7440
7441 sub windows_version {
7442     my ($ostype) = @_;
7443
7444     return 0 if !$ostype;
7445
7446     my $winversion = 0;
7447
7448     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7449         $winversion = 5;
7450     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7451         $winversion = 6;
7452     } elsif ($ostype =~ m/^win(\d+)$/) {
7453         $winversion = $1;
7454     }
7455
7456     return $winversion;
7457 }
7458
7459 sub resolve_dst_disk_format {
7460         my ($storecfg, $storeid, $src_volname, $format) = @_;
7461         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7462
7463         if (!$format) {
7464             # if no target format is specified, use the source disk format as hint
7465             if ($src_volname) {
7466                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7467                 $format = qemu_img_format($scfg, $src_volname);
7468             } else {
7469                 return $defFormat;
7470             }
7471         }
7472
7473         # test if requested format is supported - else use default
7474         my $supported = grep { $_ eq $format } @$validFormats;
7475         $format = $defFormat if !$supported;
7476         return $format;
7477 }
7478
7479 # NOTE: if this logic changes, please update docs & possibly gui logic
7480 sub find_vmstate_storage {
7481     my ($conf, $storecfg) = @_;
7482
7483     # first, return storage from conf if set
7484     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7485
7486     my ($target, $shared, $local);
7487
7488     foreach_storage_used_by_vm($conf, sub {
7489         my ($sid) = @_;
7490         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7491         my $dst = $scfg->{shared} ? \$shared : \$local;
7492         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7493     });
7494
7495     # second, use shared storage where VM has at least one disk
7496     # third, use local storage where VM has at least one disk
7497     # fall back to local storage
7498     $target = $shared // $local // 'local';
7499
7500     return $target;
7501 }
7502
7503 sub generate_uuid {
7504     my ($uuid, $uuid_str);
7505     UUID::generate($uuid);
7506     UUID::unparse($uuid, $uuid_str);
7507     return $uuid_str;
7508 }
7509
7510 sub generate_smbios1_uuid {
7511     return "uuid=".generate_uuid();
7512 }
7513
7514 sub nbd_stop {
7515     my ($vmid) = @_;
7516
7517     mon_cmd($vmid, 'nbd-server-stop');
7518 }
7519
7520 sub create_reboot_request {
7521     my ($vmid) = @_;
7522     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7523         or die "failed to create reboot trigger file: $!\n";
7524     close($fh);
7525 }
7526
7527 sub clear_reboot_request {
7528     my ($vmid) = @_;
7529     my $path = "/run/qemu-server/$vmid.reboot";
7530     my $res = 0;
7531
7532     $res = unlink($path);
7533     die "could not remove reboot request for $vmid: $!"
7534         if !$res && $! != POSIX::ENOENT;
7535
7536     return $res;
7537 }
7538
7539 sub bootorder_from_legacy {
7540     my ($conf, $bootcfg) = @_;
7541
7542     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7543     my $bootindex_hash = {};
7544     my $i = 1;
7545     foreach my $o (split(//, $boot)) {
7546         $bootindex_hash->{$o} = $i*100;
7547         $i++;
7548     }
7549
7550     my $bootorder = {};
7551
7552     PVE::QemuConfig->foreach_volume($conf, sub {
7553         my ($ds, $drive) = @_;
7554
7555         if (drive_is_cdrom ($drive, 1)) {
7556             if ($bootindex_hash->{d}) {
7557                 $bootorder->{$ds} = $bootindex_hash->{d};
7558                 $bootindex_hash->{d} += 1;
7559             }
7560         } elsif ($bootindex_hash->{c}) {
7561             $bootorder->{$ds} = $bootindex_hash->{c}
7562                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7563             $bootindex_hash->{c} += 1;
7564         }
7565     });
7566
7567     if ($bootindex_hash->{n}) {
7568         for (my $i = 0; $i < $MAX_NETS; $i++) {
7569             my $netname = "net$i";
7570             next if !$conf->{$netname};
7571             $bootorder->{$netname} = $bootindex_hash->{n};
7572             $bootindex_hash->{n} += 1;
7573         }
7574     }
7575
7576     return $bootorder;
7577 }
7578
7579 # Generate default device list for 'boot: order=' property. Matches legacy
7580 # default boot order, but with explicit device names. This is important, since
7581 # the fallback for when neither 'order' nor the old format is specified relies
7582 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7583 sub get_default_bootdevices {
7584     my ($conf) = @_;
7585
7586     my @ret = ();
7587
7588     # harddisk
7589     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7590     push @ret, $first if $first;
7591
7592     # cdrom
7593     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7594     push @ret, $first if $first;
7595
7596     # network
7597     for (my $i = 0; $i < $MAX_NETS; $i++) {
7598         my $netname = "net$i";
7599         next if !$conf->{$netname};
7600         push @ret, $netname;
7601         last;
7602     }
7603
7604     return \@ret;
7605 }
7606
7607 sub device_bootorder {
7608     my ($conf) = @_;
7609
7610     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7611
7612     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7613
7614     my $bootorder = {};
7615     if (!defined($boot) || $boot->{legacy}) {
7616         $bootorder = bootorder_from_legacy($conf, $boot);
7617     } elsif ($boot->{order}) {
7618         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7619         for my $dev (PVE::Tools::split_list($boot->{order})) {
7620             $bootorder->{$dev} = $i++;
7621         }
7622     }
7623
7624     return $bootorder;
7625 }
7626
7627 sub register_qmeventd_handle {
7628     my ($vmid) = @_;
7629
7630     my $fh;
7631     my $peer = "/var/run/qmeventd.sock";
7632     my $count = 0;
7633
7634     for (;;) {
7635         $count++;
7636         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7637         last if $fh;
7638         if ($! != EINTR && $! != EAGAIN) {
7639             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7640         }
7641         if ($count > 4) {
7642             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7643               . "after $count retries\n";
7644         }
7645         usleep(25000);
7646     }
7647
7648     # send handshake to mark VM as backing up
7649     print $fh to_json({vzdump => {vmid => "$vmid"}});
7650
7651     # return handle to be closed later when inhibit is no longer required
7652     return $fh;
7653 }
7654
7655 # bash completion helper
7656
7657 sub complete_backup_archives {
7658     my ($cmdname, $pname, $cvalue) = @_;
7659
7660     my $cfg = PVE::Storage::config();
7661
7662     my $storeid;
7663
7664     if ($cvalue =~ m/^([^:]+):/) {
7665         $storeid = $1;
7666     }
7667
7668     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7669
7670     my $res = [];
7671     foreach my $id (keys %$data) {
7672         foreach my $item (@{$data->{$id}}) {
7673             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7674             push @$res, $item->{volid} if defined($item->{volid});
7675         }
7676     }
7677
7678     return $res;
7679 }
7680
7681 my $complete_vmid_full = sub {
7682     my ($running) = @_;
7683
7684     my $idlist = vmstatus();
7685
7686     my $res = [];
7687
7688     foreach my $id (keys %$idlist) {
7689         my $d = $idlist->{$id};
7690         if (defined($running)) {
7691             next if $d->{template};
7692             next if $running && $d->{status} ne 'running';
7693             next if !$running && $d->{status} eq 'running';
7694         }
7695         push @$res, $id;
7696
7697     }
7698     return $res;
7699 };
7700
7701 sub complete_vmid {
7702     return &$complete_vmid_full();
7703 }
7704
7705 sub complete_vmid_stopped {
7706     return &$complete_vmid_full(0);
7707 }
7708
7709 sub complete_vmid_running {
7710     return &$complete_vmid_full(1);
7711 }
7712
7713 sub complete_storage {
7714
7715     my $cfg = PVE::Storage::config();
7716     my $ids = $cfg->{ids};
7717
7718     my $res = [];
7719     foreach my $sid (keys %$ids) {
7720         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7721         next if !$ids->{$sid}->{content}->{images};
7722         push @$res, $sid;
7723     }
7724
7725     return $res;
7726 }
7727
7728 sub complete_migration_storage {
7729     my ($cmd, $param, $current_value, $all_args) = @_;
7730
7731     my $targetnode = @$all_args[1];
7732
7733     my $cfg = PVE::Storage::config();
7734     my $ids = $cfg->{ids};
7735
7736     my $res = [];
7737     foreach my $sid (keys %$ids) {
7738         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7739         next if !$ids->{$sid}->{content}->{images};
7740         push @$res, $sid;
7741     }
7742
7743     return $res;
7744 }
7745
7746 sub vm_is_paused {
7747     my ($vmid) = @_;
7748     my $qmpstatus = eval {
7749         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7750         mon_cmd($vmid, "query-status");
7751     };
7752     warn "$@\n" if $@;
7753     return $qmpstatus && $qmpstatus->{status} eq "paused";
7754 }
7755
7756 sub check_volume_storage_type {
7757     my ($storecfg, $vol) = @_;
7758
7759     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7760     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7761     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7762
7763     die "storage '$storeid' does not support content-type '$vtype'\n"
7764         if !$scfg->{content}->{$vtype};
7765
7766     return 1;
7767 }
7768
7769 1;