PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Shown in the web-interface VM's summary."
 369             ." This is saved as comment inside the configuration file.",
 370         maxLength => 1024 * 8,
 371     },
 372     ostype => {
 373         optional => 1,
 374         type => 'string',
 375         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 376         description => "Specify guest operating system.",
 377         verbose_description => <<EODESC,
 378 Specify guest operating system. This is used to enable special
 379 optimization/features for specific operating systems:
 380
 381 [horizontal]
 382 other;; unspecified OS
 383 wxp;; Microsoft Windows XP
 384 w2k;; Microsoft Windows 2000
 385 w2k3;; Microsoft Windows 2003
 386 w2k8;; Microsoft Windows 2008
 387 wvista;; Microsoft Windows Vista
 388 win7;; Microsoft Windows 7
 389 win8;; Microsoft Windows 8/2012/2012r2
 390 win10;; Microsoft Windows 10/2016/2019
 391 l24;; Linux 2.4 Kernel
 392 l26;; Linux 2.6 - 5.X Kernel
 393 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 394 EODESC
 395     },
 396     boot => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-boot',
 399         description => "Specify guest boot order. Use with 'order=', usage with"
 400                      . " no key or 'legacy=' is deprecated.",
 401     },
 402     bootdisk => {
 403         optional => 1,
 404         type => 'string', format => 'pve-qm-bootdisk',
 405         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 406         pattern => '(ide|sata|scsi|virtio)\d+',
 407     },
 408     smp => {
 409         optional => 1,
 410         type => 'integer',
 411         description => "The number of CPUs. Please use option -sockets instead.",
 412         minimum => 1,
 413         default => 1,
 414     },
 415     sockets => {
 416         optional => 1,
 417         type => 'integer',
 418         description => "The number of CPU sockets.",
 419         minimum => 1,
 420         default => 1,
 421     },
 422     cores => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "The number of cores per socket.",
 426         minimum => 1,
 427         default => 1,
 428     },
 429     numa => {
 430         optional => 1,
 431         type => 'boolean',
 432         description => "Enable/disable NUMA.",
 433         default => 0,
 434     },
 435     hugepages => {
 436         optional => 1,
 437         type => 'string',
 438         description => "Enable/disable hugepages memory.",
 439         enum => [qw(any 2 1024)],
 440     },
 441     keephugepages => {
 442         optional => 1,
 443         type => 'boolean',
 444         default => 0,
 445         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 446             ." after VM shutdown and can be used for subsequent starts.",
 447     },
 448     vcpus => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "Number of hotplugged vcpus.",
 452         minimum => 1,
 453         default => 0,
 454     },
 455     acpi => {
 456         optional => 1,
 457         type => 'boolean',
 458         description => "Enable/disable ACPI.",
 459         default => 1,
 460     },
 461     agent => {
 462         optional => 1,
 463         description => "Enable/disable Qemu GuestAgent and its properties.",
 464         type => 'string',
 465         format => $agent_fmt,
 466     },
 467     kvm => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable KVM hardware virtualization.",
 471         default => 1,
 472     },
 473     tdf => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable time drift fix.",
 477         default => 0,
 478     },
 479     localtime => {
 480         optional => 1,
 481         type => 'boolean',
 482         description => "Set the real time clock to local time. This is enabled by default if ostype"
 483             ." indicates a Microsoft OS.",
 484     },
 485     freeze => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 489     },
 490     vga => {
 491         optional => 1,
 492         type => 'string', format => $vga_fmt,
 493         description => "Configure the VGA hardware.",
 494         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 495             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 496             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 497             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 498             ." display server. For win* OS you can select how many independent displays you want,"
 499             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 500             ." using a serial device as terminal.",
 501     },
 502     watchdog => {
 503         optional => 1,
 504         type => 'string', format => 'pve-qm-watchdog',
 505         description => "Create a virtual hardware watchdog device.",
 506         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 507             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 508             ." else the watchdog will reset the guest (or execute the respective action specified)",
 509     },
 510     startdate => {
 511         optional => 1,
 512         type => 'string',
 513         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 514         description => "Set the initial date of the real time clock. Valid format for date are:"
 515             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 516         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 517         default => 'now',
 518     },
 519     startup =>  get_standard_option('pve-startup-order'),
 520     template => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Enable/disable Template.",
 524         default => 0,
 525     },
 526     args => {
 527         optional => 1,
 528         type => 'string',
 529         description => "Arbitrary arguments passed to kvm.",
 530         verbose_description => <<EODESCR,
 531 Arbitrary arguments passed to kvm, for example:
 532
 533 args: -no-reboot -no-hpet
 534
 535 NOTE: this option is for experts only.
 536 EODESCR
 537     },
 538     tablet => {
 539         optional => 1,
 540         type => 'boolean',
 541         default => 1,
 542         description => "Enable/disable the USB tablet device.",
 543         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 544             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 545             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 546             ." may consider disabling this to save some context switches. This is turned off by"
 547             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 548     },
 549     migrate_speed => {
 550         optional => 1,
 551         type => 'integer',
 552         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 553         minimum => 0,
 554         default => 0,
 555     },
 556     migrate_downtime => {
 557         optional => 1,
 558         type => 'number',
 559         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 560         minimum => 0,
 561         default => 0.1,
 562     },
 563     cdrom => {
 564         optional => 1,
 565         type => 'string', format => 'pve-qm-ide',
 566         typetext => '<volume>',
 567         description => "This is an alias for option -ide2",
 568     },
 569     cpu => {
 570         optional => 1,
 571         description => "Emulated CPU type.",
 572         type => 'string',
 573         format => 'pve-vm-cpu-conf',
 574     },
 575     parent => get_standard_option('pve-snapshot-name', {
 576         optional => 1,
 577         description => "Parent snapshot name. This is used internally, and should not be modified.",
 578     }),
 579     snaptime => {
 580         optional => 1,
 581         description => "Timestamp for snapshots.",
 582         type => 'integer',
 583         minimum => 0,
 584     },
 585     vmstate => {
 586         optional => 1,
 587         type => 'string', format => 'pve-volume-id',
 588         description => "Reference to a volume which stores the VM state. This is used internally"
 589             ." for snapshots.",
 590     },
 591     vmstatestorage => get_standard_option('pve-storage-id', {
 592         description => "Default storage for VM state volumes/files.",
 593         optional => 1,
 594     }),
 595     runningmachine => get_standard_option('pve-qemu-machine', {
 596         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 597             ." for snapshots.",
 598     }),
 599     runningcpu => {
 600         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 601             ." internally for snapshots.",
 602         optional => 1,
 603         type => 'string',
 604         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 605         format_description => 'QEMU -cpu parameter'
 606     },
 607     machine => get_standard_option('pve-qemu-machine'),
 608     arch => {
 609         description => "Virtual processor architecture. Defaults to the host.",
 610         optional => 1,
 611         type => 'string',
 612         enum => [qw(x86_64 aarch64)],
 613     },
 614     smbios1 => {
 615         description => "Specify SMBIOS type 1 fields.",
 616         type => 'string', format => 'pve-qm-smbios1',
 617         maxLength => 512,
 618         optional => 1,
 619     },
 620     protection => {
 621         optional => 1,
 622         type => 'boolean',
 623         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 624             ." remove disk operations.",
 625         default => 0,
 626     },
 627     bios => {
 628         optional => 1,
 629         type => 'string',
 630         enum => [ qw(seabios ovmf) ],
 631         description => "Select BIOS implementation.",
 632         default => 'seabios',
 633     },
 634     vmgenid => {
 635         type => 'string',
 636         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 637         format_description => 'UUID',
 638         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 639             ." to disable explicitly.",
 640         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 641             ." value identifier to the guest OS. This allows to notify the guest operating system"
 642             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 643             ." execution or creation from a template). The guest operating system notices the"
 644             ." change, and is then able to react as appropriate by marking its copies of"
 645             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 646             ."Note that auto-creation only works when done through API/CLI create or update methods"
 647             .", but not when manually editing the config file.",
 648         default => "1 (autogenerated)",
 649         optional => 1,
 650     },
 651     hookscript => {
 652         type => 'string',
 653         format => 'pve-volume-id',
 654         optional => 1,
 655         description => "Script that will be executed during various steps in the vms lifetime.",
 656     },
 657     ivshmem => {
 658         type => 'string',
 659         format => $ivshmem_fmt,
 660         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 661             ." the host.",
 662         optional => 1,
 663     },
 664     audio0 => {
 665         type => 'string',
 666         format => $audio_fmt,
 667         description => "Configure a audio device, useful in combination with QXL/Spice.",
 668         optional => 1
 669     },
 670     spice_enhancements => {
 671         type => 'string',
 672         format => $spice_enhancements_fmt,
 673         description => "Configure additional enhancements for SPICE.",
 674         optional => 1
 675     },
 676     tags => {
 677         type => 'string', format => 'pve-tag-list',
 678         description => 'Tags of the VM. This is only meta information.',
 679         optional => 1,
 680     },
 681     rng0 => {
 682         type => 'string',
 683         format => $rng_fmt,
 684         description => "Configure a VirtIO-based Random Number Generator.",
 685         optional => 1,
 686     },
 687 };
 688
 689 my $cicustom_fmt = {
 690     meta => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all meta data passed to the VM via"
 694             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 695         format => 'pve-volume-id',
 696         format_description => 'volume',
 697     },
 698     network => {
 699         type => 'string',
 700         optional => 1,
 701         description => 'Specify a custom file containing all network data passed to the VM via'
 702             .' cloud-init.',
 703         format => 'pve-volume-id',
 704         format_description => 'volume',
 705     },
 706     user => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all user data passed to the VM via'
 710             .' cloud-init.',
 711         format => 'pve-volume-id',
 712         format_description => 'volume',
 713     },
 714 };
 715 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 716
 717 my $confdesc_cloudinit = {
 718     citype => {
 719         optional => 1,
 720         type => 'string',
 721         description => 'Specifies the cloud-init configuration format. The default depends on the'
 722             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 723             .' and `configdrive2` for windows.',
 724         enum => ['configdrive2', 'nocloud', 'opennebula'],
 725     },
 726     ciuser => {
 727         optional => 1,
 728         type => 'string',
 729         description => "cloud-init: User name to change ssh keys and password for instead of the"
 730             ." image's configured default user.",
 731     },
 732     cipassword => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'cloud-init: Password to assign the user. Using this is generally not'
 736             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 737             .' support hashed passwords.',
 738     },
 739     cicustom => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'cloud-init: Specify custom files to replace the automatically generated'
 743             .' ones at start.',
 744         format => 'pve-qm-cicustom',
 745     },
 746     searchdomain => {
 747         optional => 1,
 748         type => 'string',
 749         description => "cloud-init: Sets DNS search domains for a container. Create will'
 750             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 751             .' are set.",
 752     },
 753     nameserver => {
 754         optional => 1,
 755         type => 'string', format => 'address-list',
 756         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 757             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 758             .' are set.",
 759     },
 760     sshkeys => {
 761         optional => 1,
 762         type => 'string',
 763         format => 'urlencoded',
 764         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 765     },
 766 };
 767
 768 # what about other qemu settings ?
 769 #cpu => 'string',
 770 #machine => 'string',
 771 #fda => 'file',
 772 #fdb => 'file',
 773 #mtdblock => 'file',
 774 #sd => 'file',
 775 #pflash => 'file',
 776 #snapshot => 'bool',
 777 #bootp => 'file',
 778 ##tftp => 'dir',
 779 ##smb => 'dir',
 780 #kernel => 'file',
 781 #append => 'string',
 782 #initrd => 'file',
 783 ##soundhw => 'string',
 784
 785 while (my ($k, $v) = each %$confdesc) {
 786     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 787 }
 788
 789 my $MAX_USB_DEVICES = 5;
 790 my $MAX_NETS = 32;
 791 my $MAX_SERIAL_PORTS = 4;
 792 my $MAX_PARALLEL_PORTS = 3;
 793 my $MAX_NUMA = 8;
 794
 795 my $numa_fmt = {
 796     cpus => {
 797         type => "string",
 798         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 799         description => "CPUs accessing this NUMA node.",
 800         format_description => "id[-id];...",
 801     },
 802     memory => {
 803         type => "number",
 804         description => "Amount of memory this NUMA node provides.",
 805         optional => 1,
 806     },
 807     hostnodes => {
 808         type => "string",
 809         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 810         description => "Host NUMA nodes to use.",
 811         format_description => "id[-id];...",
 812         optional => 1,
 813     },
 814     policy => {
 815         type => 'string',
 816         enum => [qw(preferred bind interleave)],
 817         description => "NUMA allocation policy.",
 818         optional => 1,
 819     },
 820 };
 821 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 822 my $numadesc = {
 823     optional => 1,
 824     type => 'string', format => $numa_fmt,
 825     description => "NUMA topology.",
 826 };
 827 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 828
 829 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 830     $confdesc->{"numa$i"} = $numadesc;
 831 }
 832
 833 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 834                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 835                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 836 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 837
 838 my $net_fmt_bridge_descr = <<__EOD__;
 839 Bridge to attach the network device to. The Proxmox VE standard bridge
 840 is called 'vmbr0'.
 841
 842 If you do not specify a bridge, we create a kvm user (NATed) network
 843 device, which provides DHCP and DNS services. The following addresses
 844 are used:
 845
 846  10.0.2.2   Gateway
 847  10.0.2.3   DNS Server
 848  10.0.2.4   SMB Server
 849
 850 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 851 __EOD__
 852
 853 my $net_fmt = {
 854     macaddr  => get_standard_option('mac-addr', {
 855         description => "MAC address. That address must be unique withing your network. This is"
 856             ." automatically generated if not specified.",
 857     }),
 858     model => {
 859         type => 'string',
 860         description => "Network Card Model. The 'virtio' model provides the best performance with"
 861             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 862             ." best to use 'e1000'.",
 863         enum => $nic_model_list,
 864         default_key => 1,
 865     },
 866     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 867     bridge => {
 868         type => 'string',
 869         description => $net_fmt_bridge_descr,
 870         format_description => 'bridge',
 871         pattern => '[-_.\w\d]+',
 872         optional => 1,
 873     },
 874     queues => {
 875         type => 'integer',
 876         minimum => 0, maximum => 16,
 877         description => 'Number of packet queues to be used on the device.',
 878         optional => 1,
 879     },
 880     rate => {
 881         type => 'number',
 882         minimum => 0,
 883         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 884         optional => 1,
 885     },
 886     tag => {
 887         type => 'integer',
 888         minimum => 1, maximum => 4094,
 889         description => 'VLAN tag to apply to packets on this interface.',
 890         optional => 1,
 891     },
 892     trunks => {
 893         type => 'string',
 894         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 895         description => 'VLAN trunks to pass through this interface.',
 896         format_description => 'vlanid[;vlanid...]',
 897         optional => 1,
 898     },
 899     firewall => {
 900         type => 'boolean',
 901         description => 'Whether this interface should be protected by the firewall.',
 902         optional => 1,
 903     },
 904     link_down => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be disconnected (like pulling the plug).',
 907         optional => 1,
 908     },
 909     mtu => {
 910         type => 'integer',
 911         minimum => 1, maximum => 65520,
 912         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 913         optional => 1,
 914     },
 915 };
 916
 917 my $netdesc = {
 918     optional => 1,
 919     type => 'string', format => $net_fmt,
 920     description => "Specify network devices.",
 921 };
 922
 923 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 924
 925 my $ipconfig_fmt = {
 926     ip => {
 927         type => 'string',
 928         format => 'pve-ipv4-config',
 929         format_description => 'IPv4Format/CIDR',
 930         description => 'IPv4 address in CIDR format.',
 931         optional => 1,
 932         default => 'dhcp',
 933     },
 934     gw => {
 935         type => 'string',
 936         format => 'ipv4',
 937         format_description => 'GatewayIPv4',
 938         description => 'Default gateway for IPv4 traffic.',
 939         optional => 1,
 940         requires => 'ip',
 941     },
 942     ip6 => {
 943         type => 'string',
 944         format => 'pve-ipv6-config',
 945         format_description => 'IPv6Format/CIDR',
 946         description => 'IPv6 address in CIDR format.',
 947         optional => 1,
 948         default => 'dhcp',
 949     },
 950     gw6 => {
 951         type => 'string',
 952         format => 'ipv6',
 953         format_description => 'GatewayIPv6',
 954         description => 'Default gateway for IPv6 traffic.',
 955         optional => 1,
 956         requires => 'ip6',
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 960 my $ipconfigdesc = {
 961     optional => 1,
 962     type => 'string', format => 'pve-qm-ipconfig',
 963     description => <<'EODESCR',
 964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 965
 966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 967
 968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 969 gateway should be provided.
 970 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 971 cloud-init 19.4 or newer.
 972
 973 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 974 dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $usb_fmt = {
1006     host => {
1007         default_key => 1,
1008         type => 'string', format => 'pve-qm-usb-device',
1009         format_description => 'HOSTUSBDEVICE|spice',
1010         description => <<EODESCR,
1011 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1012
1013  'bus-port(.port)*' (decimal numbers) or
1014  'vendor_id:product_id' (hexadeciaml numbers) or
1015  'spice'
1016
1017 You can use the 'lsusb -t' command to list existing usb devices.
1018
1019 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1020 machines - use with special care.
1021
1022 The value 'spice' can be used to add a usb redirection devices for spice.
1023 EODESCR
1024     },
1025     usb3 => {
1026         optional => 1,
1027         type => 'boolean',
1028         description => "Specifies whether if given host option is a USB3 device or port.",
1029         default => 0,
1030     },
1031 };
1032
1033 my $usbdesc = {
1034     optional => 1,
1035     type => 'string', format => $usb_fmt,
1036     description => "Configure an USB device (n is 0 to 4).",
1037 };
1038 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1039
1040 my $serialdesc = {
1041         optional => 1,
1042         type => 'string',
1043         pattern => '(/dev/.+|socket)',
1044         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1045         verbose_description =>  <<EODESCR,
1046 Create a serial device inside the VM (n is 0 to 3), and pass through a
1047 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1048 host side (use 'qm terminal' to open a terminal connection).
1049
1050 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1051 use with special care.
1052
1053 CAUTION: Experimental! User reported problems with this option.
1054 EODESCR
1055 };
1056
1057 my $paralleldesc= {
1058         optional => 1,
1059         type => 'string',
1060         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1061         description =>  "Map host parallel devices (n is 0 to 2).",
1062         verbose_description =>  <<EODESCR,
1063 Map host parallel devices (n is 0 to 2).
1064
1065 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1066 machines - use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1073     $confdesc->{"parallel$i"} = $paralleldesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1077     $confdesc->{"serial$i"} = $serialdesc;
1078 }
1079
1080 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1081     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1082 }
1083
1084 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1085     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1086 }
1087
1088 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1089     $confdesc->{"usb$i"} = $usbdesc;
1090 }
1091
1092 my $boot_fmt = {
1093     legacy => {
1094         optional => 1,
1095         default_key => 1,
1096         type => 'string',
1097         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1098                      . " Deprecated, use 'order=' instead.",
1099         pattern => '[acdn]{1,4}',
1100         format_description => "[acdn]{1,4}",
1101
1102         # note: this is also the fallback if boot: is not given at all
1103         default => 'cdn',
1104     },
1105     order => {
1106         optional => 1,
1107         type => 'string',
1108         format => 'pve-qm-bootdev-list',
1109         format_description => "device[;device...]",
1110         description => <<EODESC,
1111 The guest will attempt to boot from devices in the order they appear here.
1112
1113 Disks, optical drives and passed-through storage USB devices will be directly
1114 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1115 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1116
1117 Note that only devices in this list will be marked as bootable and thus loaded
1118 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1119 (e.g. software-raid), you need to specify all of them here.
1120
1121 Overrides the deprecated 'legacy=[acdn]*' value when given.
1122 EODESC
1123     },
1124 };
1125 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1126
1127 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1128 sub verify_bootdev {
1129     my ($dev, $noerr) = @_;
1130
1131     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1132
1133     my $check = sub {
1134         my ($base) = @_;
1135         return 0 if $dev !~ m/^$base\d+$/;
1136         return 0 if !$confdesc->{$dev};
1137         return 1;
1138     };
1139
1140     return $dev if $check->("net");
1141     return $dev if $check->("usb");
1142     return $dev if $check->("hostpci");
1143
1144     return if $noerr;
1145     die "invalid boot device '$dev'\n";
1146 }
1147
1148 sub print_bootorder {
1149     my ($devs) = @_;
1150     return "" if !@$devs;
1151     my $data = { order => join(';', @$devs) };
1152     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1153 }
1154
1155 my $kvm_api_version = 0;
1156
1157 sub kvm_version {
1158     return $kvm_api_version if $kvm_api_version;
1159
1160     open my $fh, '<', '/dev/kvm' or return;
1161
1162     # 0xae00 => KVM_GET_API_VERSION
1163     $kvm_api_version = ioctl($fh, 0xae00, 0);
1164     close($fh);
1165
1166     return $kvm_api_version;
1167 }
1168
1169 my $kvm_user_version = {};
1170 my $kvm_mtime = {};
1171
1172 sub kvm_user_version {
1173     my ($binary) = @_;
1174
1175     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1176     my $st = stat($binary);
1177
1178     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180         $cachedmtime == $st->mtime;
1181
1182     $kvm_user_version->{$binary} = 'unknown';
1183     $kvm_mtime->{$binary} = $st->mtime;
1184
1185     my $code = sub {
1186         my $line = shift;
1187         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188             $kvm_user_version->{$binary} = $2;
1189         }
1190     };
1191
1192     eval { run_command([$binary, '--version'], outfunc => $code); };
1193     warn $@ if $@;
1194
1195     return $kvm_user_version->{$binary};
1196
1197 }
1198 my sub extract_version {
1199     my ($machine_type, $version) = @_;
1200     $version = kvm_user_version() if !defined($version);
1201     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1202 }
1203
1204 sub kernel_has_vhost_net {
1205     return -c '/dev/vhost-net';
1206 }
1207
1208 sub option_exists {
1209     my $key = shift;
1210     return defined($confdesc->{$key});
1211 }
1212
1213 my $cdrom_path;
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241      if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1286             if !$vtype;
1287         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1288         verify_media_type($opt, $vtype, $drive->{media});
1289         $drive->{file} = $volid;
1290     }
1291
1292     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1293 }
1294
1295 sub parse_hotplug_features {
1296     my ($data) = @_;
1297
1298     my $res = {};
1299
1300     return $res if $data eq '0';
1301
1302     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1303
1304     foreach my $feature (PVE::Tools::split_list($data)) {
1305         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306             $res->{$1} = 1;
1307         } else {
1308             die "invalid hotplug feature '$feature'\n";
1309         }
1310     }
1311     return $res;
1312 }
1313
1314 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1315 sub pve_verify_hotplug_features {
1316     my ($value, $noerr) = @_;
1317
1318     return $value if parse_hotplug_features($value);
1319
1320     return if $noerr;
1321
1322     die "unable to parse hotplug option\n";
1323 }
1324
1325 sub scsi_inquiry {
1326     my($fh, $noerr) = @_;
1327
1328     my $SG_IO = 0x2285;
1329     my $SG_GET_VERSION_NUM = 0x2282;
1330
1331     my $versionbuf = "\x00" x 8;
1332     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333     if (!$ret) {
1334         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335         return;
1336     }
1337     my $version = unpack("I", $versionbuf);
1338     if ($version < 30000) {
1339         die "scsi generic interface too old\n"  if !$noerr;
1340         return;
1341     }
1342
1343     my $buf = "\x00" x 36;
1344     my $sensebuf = "\x00" x 8;
1345     my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347     # see /usr/include/scsi/sg.h
1348     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351                       length($sensebuf), 0, length($buf), $buf,
1352                       $cmd, $sensebuf, 6000);
1353
1354     $ret = ioctl($fh, $SG_IO, $packet);
1355     if (!$ret) {
1356         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357         return;
1358     }
1359
1360     my @res = unpack($sg_io_hdr_t, $packet);
1361     if ($res[17] || $res[18]) {
1362         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363         return;
1364     }
1365
1366     my $res = {};
1367     (my $byte0, my $byte1, $res->{vendor},
1368      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370     $res->{removable} = $byte1 & 128 ? 1 : 0;
1371     $res->{type} = $byte0 & 31;
1372
1373     return $res;
1374 }
1375
1376 sub path_is_scsi {
1377     my ($path) = @_;
1378
1379     my $fh = IO::File->new("+<$path") || return;
1380     my $res = scsi_inquiry($fh, 1);
1381     close($fh);
1382
1383     return $res;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf, $arch) = @_;
1388
1389     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus;
1393     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1394         $usbbus = 'ehci';
1395     } else {
1396         $usbbus = 'uhci';
1397     }
1398
1399     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400 }
1401
1402 sub print_keyboarddevice_full {
1403     my ($conf, $arch, $machine) = @_;
1404
1405     return if $arch ne 'aarch64';
1406
1407     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408 }
1409
1410 my sub get_drive_id {
1411     my ($drive) = @_;
1412     return "$drive->{interface}$drive->{index}";
1413 }
1414
1415 sub print_drivedevice_full {
1416     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1417
1418     my $device = '';
1419     my $maxdev = 0;
1420
1421     my $drive_id = get_drive_id($drive);
1422     if ($drive->{interface} eq 'virtio') {
1423         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1424         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1425         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1426     } elsif ($drive->{interface} eq 'scsi') {
1427
1428         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1429         my $unit = $drive->{index} % $maxdev;
1430         my $devicetype = 'hd';
1431         my $path = '';
1432         if (drive_is_cdrom($drive)) {
1433             $devicetype = 'cd';
1434         } else {
1435             if ($drive->{file} =~ m|^/|) {
1436                 $path = $drive->{file};
1437                 if (my $info = path_is_scsi($path)) {
1438                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1439                         $devicetype = 'block';
1440                     } elsif ($info->{type} == 1) { # tape
1441                         $devicetype = 'generic';
1442                     }
1443                 }
1444             } else {
1445                  $path = PVE::Storage::path($storecfg, $drive->{file});
1446             }
1447
1448             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1449             my $version = extract_version($machine_type, kvm_user_version());
1450             if ($path =~ m/^iscsi\:\/\// &&
1451                !min_version($version, 4, 1)) {
1452                 $devicetype = 'generic';
1453             }
1454         }
1455
1456         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1458         } else {
1459             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1460                 .",lun=$drive->{index}";
1461         }
1462         $device .= ",drive=drive-$drive_id,id=$drive_id";
1463
1464         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1465             $device .= ",rotation_rate=1";
1466         }
1467         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1468
1469     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1470         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1471         my $controller = int($drive->{index} / $maxdev);
1472         my $unit = $drive->{index} % $maxdev;
1473         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1474
1475         $device = "ide-$devicetype";
1476         if ($drive->{interface} eq 'ide') {
1477             $device .= ",bus=ide.$controller,unit=$unit";
1478         } else {
1479             $device .= ",bus=ahci$controller.$unit";
1480         }
1481         $device .= ",drive=drive-$drive_id,id=$drive_id";
1482
1483         if ($devicetype eq 'hd') {
1484             if (my $model = $drive->{model}) {
1485                 $model = URI::Escape::uri_unescape($model);
1486                 $device .= ",model=$model";
1487             }
1488             if ($drive->{ssd}) {
1489                 $device .= ",rotation_rate=1";
1490             }
1491         }
1492         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1493     } elsif ($drive->{interface} eq 'usb') {
1494         die "implement me";
1495         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1496     } else {
1497         die "unsupported interface type";
1498     }
1499
1500     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1501
1502     if (my $serial = $drive->{serial}) {
1503         $serial = URI::Escape::uri_unescape($serial);
1504         $device .= ",serial=$serial";
1505     }
1506
1507
1508     return $device;
1509 }
1510
1511 sub get_initiator_name {
1512     my $initiator;
1513
1514     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1515     while (defined(my $line = <$fh>)) {
1516         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1517         $initiator = $1;
1518         last;
1519     }
1520     $fh->close();
1521
1522     return $initiator;
1523 }
1524
1525 sub print_drive_commandline_full {
1526     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1527
1528     my $path;
1529     my $volid = $drive->{file};
1530     my $format = $drive->{format};
1531     my $drive_id = get_drive_id($drive);
1532
1533     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1534     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1535
1536     if (drive_is_cdrom($drive)) {
1537         $path = get_iso_path($storecfg, $vmid, $volid);
1538         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1539     } else {
1540         if ($storeid) {
1541             $path = PVE::Storage::path($storecfg, $volid);
1542             $format //= qemu_img_format($scfg, $volname);
1543         } else {
1544             $path = $volid;
1545             $format //= "raw";
1546         }
1547    }
1548
1549    my $is_rbd = $path =~ m/^rbd:/;
1550
1551     my $opts = '';
1552     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1553     foreach my $o (@qemu_drive_options) {
1554         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1555     }
1556
1557     # snapshot only accepts on|off
1558     if (defined($drive->{snapshot})) {
1559         my $v = $drive->{snapshot} ? 'on' : 'off';
1560         $opts .= ",snapshot=$v";
1561     }
1562
1563     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1564         my ($dir, $qmpname) = @$type;
1565         if (my $v = $drive->{"mbps$dir"}) {
1566             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1567         }
1568         if (my $v = $drive->{"mbps${dir}_max"}) {
1569             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1570         }
1571         if (my $v = $drive->{"bps${dir}_max_length"}) {
1572             $opts .= ",throttling.bps$qmpname-max-length=$v";
1573         }
1574         if (my $v = $drive->{"iops${dir}"}) {
1575             $opts .= ",throttling.iops$qmpname=$v";
1576         }
1577         if (my $v = $drive->{"iops${dir}_max"}) {
1578             $opts .= ",throttling.iops$qmpname-max=$v";
1579         }
1580         if (my $v = $drive->{"iops${dir}_max_length"}) {
1581             $opts .= ",throttling.iops$qmpname-max-length=$v";
1582         }
1583     }
1584
1585     if ($pbs_name) {
1586         $format = "rbd" if $is_rbd;
1587         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1588             if !$format;
1589         $opts .= ",format=alloc-track,file.driver=$format";
1590     } elsif ($format) {
1591         $opts .= ",format=$format";
1592     }
1593
1594     my $cache_direct = 0;
1595
1596     if (my $cache = $drive->{cache}) {
1597         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1598     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1599         $opts .= ",cache=none";
1600         $cache_direct = 1;
1601     }
1602
1603     if (!$drive->{aio}) {
1604         if ($io_uring) {
1605             # io_uring supports all cache modes
1606             $opts .= ",aio=io_uring";
1607         } else {
1608             # aio native works only with O_DIRECT
1609             if($cache_direct) {
1610                 $opts .= ",aio=native";
1611             } else {
1612                 $opts .= ",aio=threads";
1613             }
1614         }
1615     }
1616
1617     if (!drive_is_cdrom($drive)) {
1618         my $detectzeroes;
1619         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1620             $detectzeroes = 'off';
1621         } elsif ($drive->{discard}) {
1622             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1623         } else {
1624             # This used to be our default with discard not being specified:
1625             $detectzeroes = 'on';
1626         }
1627
1628         # note: 'detect-zeroes' works per blockdev and we want it to persist
1629         # after the alloc-track is removed, so put it on 'file' directly
1630         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1631         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1632     }
1633
1634     if ($pbs_name) {
1635         $opts .= ",backing=$pbs_name";
1636         $opts .= ",auto-remove=on";
1637     }
1638
1639     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1640     my $file_param = "file";
1641     if ($pbs_name) {
1642         # non-rbd drivers require the underlying file to be a seperate block
1643         # node, so add a second .file indirection
1644         $file_param .= ".file" if !$is_rbd;
1645         $file_param .= ".filename";
1646     }
1647     my $pathinfo = $path ? "$file_param=$path," : '';
1648
1649     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1650 }
1651
1652 sub print_pbs_blockdev {
1653     my ($pbs_conf, $pbs_name) = @_;
1654     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1655     $blockdev .= ",repository=$pbs_conf->{repository}";
1656     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1657     $blockdev .= ",archive=$pbs_conf->{archive}";
1658     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1659     return $blockdev;
1660 }
1661
1662 sub print_netdevice_full {
1663     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1664
1665     my $device = $net->{model};
1666     if ($net->{model} eq 'virtio') {
1667          $device = 'virtio-net-pci';
1668      };
1669
1670     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1671     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1672     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1673         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1674         # and out of each queue plus one config interrupt and control vector queue
1675         my $vectors = $net->{queues} * 2 + 2;
1676         $tmpstr .= ",vectors=$vectors,mq=on";
1677     }
1678     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1679
1680     if (my $mtu = $net->{mtu}) {
1681         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1682             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1683             if ($mtu == 1) {
1684                  $mtu = $bridge_mtu;
1685             } elsif ($mtu < 576) {
1686                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1687             } elsif ($mtu > $bridge_mtu) {
1688                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1689             }
1690             $tmpstr .= ",host_mtu=$mtu";
1691         } else {
1692             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1693         }
1694     }
1695
1696     if ($use_old_bios_files) {
1697         my $romfile;
1698         if ($device eq 'virtio-net-pci') {
1699             $romfile = 'pxe-virtio.rom';
1700         } elsif ($device eq 'e1000') {
1701             $romfile = 'pxe-e1000.rom';
1702         } elsif ($device eq 'ne2k') {
1703             $romfile = 'pxe-ne2k_pci.rom';
1704         } elsif ($device eq 'pcnet') {
1705             $romfile = 'pxe-pcnet.rom';
1706         } elsif ($device eq 'rtl8139') {
1707             $romfile = 'pxe-rtl8139.rom';
1708         }
1709         $tmpstr .= ",romfile=$romfile" if $romfile;
1710     }
1711
1712     return $tmpstr;
1713 }
1714
1715 sub print_netdev_full {
1716     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1717
1718     my $i = '';
1719     if ($netid =~ m/^net(\d+)$/) {
1720         $i = int($1);
1721     }
1722
1723     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1724
1725     my $ifname = "tap${vmid}i$i";
1726
1727     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1728     die "interface name '$ifname' is too long (max 15 character)\n"
1729         if length($ifname) >= 16;
1730
1731     my $vhostparam = '';
1732     if (is_native($arch)) {
1733         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1734     }
1735
1736     my $vmname = $conf->{name} || "vm$vmid";
1737
1738     my $netdev = "";
1739     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1740
1741     if ($net->{bridge}) {
1742         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1743             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1744     } else {
1745         $netdev = "type=user,id=$netid,hostname=$vmname";
1746     }
1747
1748     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1749
1750     return $netdev;
1751 }
1752
1753 my $vga_map = {
1754     'cirrus' => 'cirrus-vga',
1755     'std' => 'VGA',
1756     'vmware' => 'vmware-svga',
1757     'virtio' => 'virtio-vga',
1758 };
1759
1760 sub print_vga_device {
1761     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1762
1763     my $type = $vga_map->{$vga->{type}};
1764     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1765         $type = 'virtio-gpu';
1766     }
1767     my $vgamem_mb = $vga->{memory};
1768
1769     my $max_outputs = '';
1770     if ($qxlnum) {
1771         $type = $id ? 'qxl' : 'qxl-vga';
1772
1773         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1774             # set max outputs so linux can have up to 4 qxl displays with one device
1775             if (min_version($machine_version, 4, 1)) {
1776                 $max_outputs = ",max_outputs=4";
1777             }
1778         }
1779     }
1780
1781     die "no devicetype for $vga->{type}\n" if !$type;
1782
1783     my $memory = "";
1784     if ($vgamem_mb) {
1785         if ($vga->{type} eq 'virtio') {
1786             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1787             $memory = ",max_hostmem=$bytes";
1788         } elsif ($qxlnum) {
1789             # from https://www.spice-space.org/multiple-monitors.html
1790             $memory = ",vgamem_mb=$vga->{memory}";
1791             my $ram = $vgamem_mb * 4;
1792             my $vram = $vgamem_mb * 2;
1793             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1794         } else {
1795             $memory = ",vgamem_mb=$vga->{memory}";
1796         }
1797     } elsif ($qxlnum && $id) {
1798         $memory = ",ram_size=67108864,vram_size=33554432";
1799     }
1800
1801     my $edidoff = "";
1802     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1803         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1804     }
1805
1806     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1807     my $vgaid = "vga" . ($id // '');
1808     my $pciaddr;
1809
1810     if ($q35 && $vgaid eq 'vga') {
1811         # the first display uses pcie.0 bus on q35 machines
1812         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1813     } else {
1814         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1815     }
1816
1817     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1818 }
1819
1820 sub parse_number_sets {
1821     my ($set) = @_;
1822     my $res = [];
1823     foreach my $part (split(/;/, $set)) {
1824         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1825             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1826             push @$res, [ $1, $2 ];
1827         } else {
1828             die "invalid range: $part\n";
1829         }
1830     }
1831     return $res;
1832 }
1833
1834 sub parse_numa {
1835     my ($data) = @_;
1836
1837     my $res = parse_property_string($numa_fmt, $data);
1838     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1839     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1840     return $res;
1841 }
1842
1843 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1844 sub parse_net {
1845     my ($data) = @_;
1846
1847     my $res = eval { parse_property_string($net_fmt, $data) };
1848     if ($@) {
1849         warn $@;
1850         return;
1851     }
1852     if (!defined($res->{macaddr})) {
1853         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1854         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1855     }
1856     return $res;
1857 }
1858
1859 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1860 sub parse_ipconfig {
1861     my ($data) = @_;
1862
1863     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1864     if ($@) {
1865         warn $@;
1866         return;
1867     }
1868
1869     if ($res->{gw} && !$res->{ip}) {
1870         warn 'gateway specified without specifying an IP address';
1871         return;
1872     }
1873     if ($res->{gw6} && !$res->{ip6}) {
1874         warn 'IPv6 gateway specified without specifying an IPv6 address';
1875         return;
1876     }
1877     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1878         warn 'gateway specified together with DHCP';
1879         return;
1880     }
1881     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1882         # gw6 + auto/dhcp
1883         warn "IPv6 gateway specified together with $res->{ip6} address";
1884         return;
1885     }
1886
1887     if (!$res->{ip} && !$res->{ip6}) {
1888         return { ip => 'dhcp', ip6 => 'dhcp' };
1889     }
1890
1891     return $res;
1892 }
1893
1894 sub print_net {
1895     my $net = shift;
1896
1897     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1898 }
1899
1900 sub add_random_macs {
1901     my ($settings) = @_;
1902
1903     foreach my $opt (keys %$settings) {
1904         next if $opt !~ m/^net(\d+)$/;
1905         my $net = parse_net($settings->{$opt});
1906         next if !$net;
1907         $settings->{$opt} = print_net($net);
1908     }
1909 }
1910
1911 sub vm_is_volid_owner {
1912     my ($storecfg, $vmid, $volid) = @_;
1913
1914     if ($volid !~  m|^/|) {
1915         my ($path, $owner);
1916         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1917         if ($owner && ($owner == $vmid)) {
1918             return 1;
1919         }
1920     }
1921
1922     return;
1923 }
1924
1925 sub vmconfig_register_unused_drive {
1926     my ($storecfg, $vmid, $conf, $drive) = @_;
1927
1928     if (drive_is_cloudinit($drive)) {
1929         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1930         warn $@ if $@;
1931     } elsif (!drive_is_cdrom($drive)) {
1932         my $volid = $drive->{file};
1933         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1934             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1935         }
1936     }
1937 }
1938
1939 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1940 my $smbios1_fmt = {
1941     uuid => {
1942         type => 'string',
1943         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1944         format_description => 'UUID',
1945         description => "Set SMBIOS1 UUID.",
1946         optional => 1,
1947     },
1948     version => {
1949         type => 'string',
1950         pattern => '[A-Za-z0-9+\/]+={0,2}',
1951         format_description => 'Base64 encoded string',
1952         description => "Set SMBIOS1 version.",
1953         optional => 1,
1954     },
1955     serial => {
1956         type => 'string',
1957         pattern => '[A-Za-z0-9+\/]+={0,2}',
1958         format_description => 'Base64 encoded string',
1959         description => "Set SMBIOS1 serial number.",
1960         optional => 1,
1961     },
1962     manufacturer => {
1963         type => 'string',
1964         pattern => '[A-Za-z0-9+\/]+={0,2}',
1965         format_description => 'Base64 encoded string',
1966         description => "Set SMBIOS1 manufacturer.",
1967         optional => 1,
1968     },
1969     product => {
1970         type => 'string',
1971         pattern => '[A-Za-z0-9+\/]+={0,2}',
1972         format_description => 'Base64 encoded string',
1973         description => "Set SMBIOS1 product ID.",
1974         optional => 1,
1975     },
1976     sku => {
1977         type => 'string',
1978         pattern => '[A-Za-z0-9+\/]+={0,2}',
1979         format_description => 'Base64 encoded string',
1980         description => "Set SMBIOS1 SKU string.",
1981         optional => 1,
1982     },
1983     family => {
1984         type => 'string',
1985         pattern => '[A-Za-z0-9+\/]+={0,2}',
1986         format_description => 'Base64 encoded string',
1987         description => "Set SMBIOS1 family string.",
1988         optional => 1,
1989     },
1990     base64 => {
1991         type => 'boolean',
1992         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1993         optional => 1,
1994     },
1995 };
1996
1997 sub parse_smbios1 {
1998     my ($data) = @_;
1999
2000     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2001     warn $@ if $@;
2002     return $res;
2003 }
2004
2005 sub print_smbios1 {
2006     my ($smbios1) = @_;
2007     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2008 }
2009
2010 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2011
2012 sub parse_watchdog {
2013     my ($value) = @_;
2014
2015     return if !$value;
2016
2017     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2018     warn $@ if $@;
2019     return $res;
2020 }
2021
2022 sub parse_guest_agent {
2023     my ($conf) = @_;
2024
2025     return {} if !defined($conf->{agent});
2026
2027     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2028     warn $@ if $@;
2029
2030     # if the agent is disabled ignore the other potentially set properties
2031     return {} if !$res->{enabled};
2032     return $res;
2033 }
2034
2035 sub get_qga_key {
2036     my ($conf, $key) = @_;
2037     return undef if !defined($conf->{agent});
2038
2039     my $agent = parse_guest_agent($conf);
2040     return $agent->{$key};
2041 }
2042
2043 sub parse_vga {
2044     my ($value) = @_;
2045
2046     return {} if !$value;
2047     my $res = eval { parse_property_string($vga_fmt, $value) };
2048     warn $@ if $@;
2049     return $res;
2050 }
2051
2052 sub parse_rng {
2053     my ($value) = @_;
2054
2055     return if !$value;
2056
2057     my $res = eval { parse_property_string($rng_fmt, $value) };
2058     warn $@ if $@;
2059     return $res;
2060 }
2061
2062 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2063 sub verify_usb_device {
2064     my ($value, $noerr) = @_;
2065
2066     return $value if parse_usb_device($value);
2067
2068     return if $noerr;
2069
2070     die "unable to parse usb device\n";
2071 }
2072
2073 # add JSON properties for create and set function
2074 sub json_config_properties {
2075     my $prop = shift;
2076
2077     foreach my $opt (keys %$confdesc) {
2078         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2079             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2080         $prop->{$opt} = $confdesc->{$opt};
2081     }
2082
2083     return $prop;
2084 }
2085
2086 # return copy of $confdesc_cloudinit to generate documentation
2087 sub cloudinit_config_properties {
2088
2089     return dclone($confdesc_cloudinit);
2090 }
2091
2092 sub check_type {
2093     my ($key, $value) = @_;
2094
2095     die "unknown setting '$key'\n" if !$confdesc->{$key};
2096
2097     my $type = $confdesc->{$key}->{type};
2098
2099     if (!defined($value)) {
2100         die "got undefined value\n";
2101     }
2102
2103     if ($value =~ m/[\n\r]/) {
2104         die "property contains a line feed\n";
2105     }
2106
2107     if ($type eq 'boolean') {
2108         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2109         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2110         die "type check ('boolean') failed - got '$value'\n";
2111     } elsif ($type eq 'integer') {
2112         return int($1) if $value =~ m/^(\d+)$/;
2113         die "type check ('integer') failed - got '$value'\n";
2114     } elsif ($type eq 'number') {
2115         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2116         die "type check ('number') failed - got '$value'\n";
2117     } elsif ($type eq 'string') {
2118         if (my $fmt = $confdesc->{$key}->{format}) {
2119             PVE::JSONSchema::check_format($fmt, $value);
2120             return $value;
2121         }
2122         $value =~ s/^\"(.*)\"$/$1/;
2123         return $value;
2124     } else {
2125         die "internal error"
2126     }
2127 }
2128
2129 sub destroy_vm {
2130     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2131
2132     my $conf = PVE::QemuConfig->load_config($vmid);
2133
2134     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2135
2136     if ($conf->{template}) {
2137         # check if any base image is still used by a linked clone
2138         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2139                 my ($ds, $drive) = @_;
2140                 return if drive_is_cdrom($drive);
2141
2142                 my $volid = $drive->{file};
2143                 return if !$volid || $volid =~ m|^/|;
2144
2145                 die "base volume '$volid' is still in use by linked cloned\n"
2146                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2147
2148         });
2149     }
2150
2151     my $remove_owned_drive = sub {
2152         my ($ds, $drive) = @_;
2153         return if drive_is_cdrom($drive, 1);
2154
2155         my $volid = $drive->{file};
2156         return if !$volid || $volid =~ m|^/|;
2157
2158         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2159         return if !$path || !$owner || ($owner != $vmid);
2160
2161         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2162         warn "Could not remove disk '$volid', check manually: $@" if $@;
2163     };
2164
2165     # only remove disks owned by this VM (referenced in the config)
2166     my $include_opts = {
2167         include_unused => 1,
2168         extra_keys => ['vmstate'],
2169     };
2170     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2171
2172     for my $snap (values %{$conf->{snapshots}}) {
2173         next if !defined($snap->{vmstate});
2174         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2175         next if !defined($drive);
2176         $remove_owned_drive->('vmstate', $drive);
2177     }
2178
2179     if ($purge_unreferenced) { # also remove unreferenced disk
2180         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2181         PVE::Storage::foreach_volid($vmdisks, sub {
2182             my ($volid, $sid, $volname, $d) = @_;
2183             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2184             warn $@ if $@;
2185         });
2186     }
2187
2188     if (defined $replacement_conf) {
2189         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2190     } else {
2191         PVE::QemuConfig->destroy_config($vmid);
2192     }
2193 }
2194
2195 sub parse_vm_config {
2196     my ($filename, $raw) = @_;
2197
2198     return if !defined($raw);
2199
2200     my $res = {
2201         digest => Digest::SHA::sha1_hex($raw),
2202         snapshots => {},
2203         pending => {},
2204     };
2205
2206     $filename =~ m|/qemu-server/(\d+)\.conf$|
2207         || die "got strange filename '$filename'";
2208
2209     my $vmid = $1;
2210
2211     my $conf = $res;
2212     my $descr;
2213     my $section = '';
2214
2215     my @lines = split(/\n/, $raw);
2216     foreach my $line (@lines) {
2217         next if $line =~ m/^\s*$/;
2218
2219         if ($line =~ m/^\[PENDING\]\s*$/i) {
2220             $section = 'pending';
2221             if (defined($descr)) {
2222                 $descr =~ s/\s+$//;
2223                 $conf->{description} = $descr;
2224             }
2225             $descr = undef;
2226             $conf = $res->{$section} = {};
2227             next;
2228
2229         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2230             $section = $1;
2231             if (defined($descr)) {
2232                 $descr =~ s/\s+$//;
2233                 $conf->{description} = $descr;
2234             }
2235             $descr = undef;
2236             $conf = $res->{snapshots}->{$section} = {};
2237             next;
2238         }
2239
2240         if ($line =~ m/^\#(.*)\s*$/) {
2241             $descr = '' if !defined($descr);
2242             $descr .= PVE::Tools::decode_text($1) . "\n";
2243             next;
2244         }
2245
2246         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2247             $descr = '' if !defined($descr);
2248             $descr .= PVE::Tools::decode_text($2);
2249         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2250             $conf->{snapstate} = $1;
2251         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2252             my $key = $1;
2253             my $value = $2;
2254             $conf->{$key} = $value;
2255         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2256             my $value = $1;
2257             if ($section eq 'pending') {
2258                 $conf->{delete} = $value; # we parse this later
2259             } else {
2260                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2261             }
2262         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2263             my $key = $1;
2264             my $value = $2;
2265             eval { $value = check_type($key, $value); };
2266             if ($@) {
2267                 warn "vm $vmid - unable to parse value of '$key' - $@";
2268             } else {
2269                 $key = 'ide2' if $key eq 'cdrom';
2270                 my $fmt = $confdesc->{$key}->{format};
2271                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2272                     my $v = parse_drive($key, $value);
2273                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2274                         $v->{file} = $volid;
2275                         $value = print_drive($v);
2276                     } else {
2277                         warn "vm $vmid - unable to parse value of '$key'\n";
2278                         next;
2279                     }
2280                 }
2281
2282                 $conf->{$key} = $value;
2283             }
2284         } else {
2285             warn "vm $vmid - unable to parse config: $line\n";
2286         }
2287     }
2288
2289     if (defined($descr)) {
2290         $descr =~ s/\s+$//;
2291         $conf->{description} = $descr;
2292     }
2293     delete $res->{snapstate}; # just to be sure
2294
2295     return $res;
2296 }
2297
2298 sub write_vm_config {
2299     my ($filename, $conf) = @_;
2300
2301     delete $conf->{snapstate}; # just to be sure
2302
2303     if ($conf->{cdrom}) {
2304         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2305         $conf->{ide2} = $conf->{cdrom};
2306         delete $conf->{cdrom};
2307     }
2308
2309     # we do not use 'smp' any longer
2310     if ($conf->{sockets}) {
2311         delete $conf->{smp};
2312     } elsif ($conf->{smp}) {
2313         $conf->{sockets} = $conf->{smp};
2314         delete $conf->{cores};
2315         delete $conf->{smp};
2316     }
2317
2318     my $used_volids = {};
2319
2320     my $cleanup_config = sub {
2321         my ($cref, $pending, $snapname) = @_;
2322
2323         foreach my $key (keys %$cref) {
2324             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2325                 $key eq 'snapstate' || $key eq 'pending';
2326             my $value = $cref->{$key};
2327             if ($key eq 'delete') {
2328                 die "propertry 'delete' is only allowed in [PENDING]\n"
2329                     if !$pending;
2330                 # fixme: check syntax?
2331                 next;
2332             }
2333             eval { $value = check_type($key, $value); };
2334             die "unable to parse value of '$key' - $@" if $@;
2335
2336             $cref->{$key} = $value;
2337
2338             if (!$snapname && is_valid_drivename($key)) {
2339                 my $drive = parse_drive($key, $value);
2340                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2341             }
2342         }
2343     };
2344
2345     &$cleanup_config($conf);
2346
2347     &$cleanup_config($conf->{pending}, 1);
2348
2349     foreach my $snapname (keys %{$conf->{snapshots}}) {
2350         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2351         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2352     }
2353
2354     # remove 'unusedX' settings if we re-add a volume
2355     foreach my $key (keys %$conf) {
2356         my $value = $conf->{$key};
2357         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2358             delete $conf->{$key};
2359         }
2360     }
2361
2362     my $generate_raw_config = sub {
2363         my ($conf, $pending) = @_;
2364
2365         my $raw = '';
2366
2367         # add description as comment to top of file
2368         if (defined(my $descr = $conf->{description})) {
2369             if ($descr) {
2370                 foreach my $cl (split(/\n/, $descr)) {
2371                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2372                 }
2373             } else {
2374                 $raw .= "#\n" if $pending;
2375             }
2376         }
2377
2378         foreach my $key (sort keys %$conf) {
2379             next if $key =~ /^(digest|description|pending|snapshots)$/;
2380             $raw .= "$key: $conf->{$key}\n";
2381         }
2382         return $raw;
2383     };
2384
2385     my $raw = &$generate_raw_config($conf);
2386
2387     if (scalar(keys %{$conf->{pending}})){
2388         $raw .= "\n[PENDING]\n";
2389         $raw .= &$generate_raw_config($conf->{pending}, 1);
2390     }
2391
2392     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2393         $raw .= "\n[$snapname]\n";
2394         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2395     }
2396
2397     return $raw;
2398 }
2399
2400 sub load_defaults {
2401
2402     my $res = {};
2403
2404     # we use static defaults from our JSON schema configuration
2405     foreach my $key (keys %$confdesc) {
2406         if (defined(my $default = $confdesc->{$key}->{default})) {
2407             $res->{$key} = $default;
2408         }
2409     }
2410
2411     return $res;
2412 }
2413
2414 sub config_list {
2415     my $vmlist = PVE::Cluster::get_vmlist();
2416     my $res = {};
2417     return $res if !$vmlist || !$vmlist->{ids};
2418     my $ids = $vmlist->{ids};
2419     my $nodename = nodename();
2420
2421     foreach my $vmid (keys %$ids) {
2422         my $d = $ids->{$vmid};
2423         next if !$d->{node} || $d->{node} ne $nodename;
2424         next if !$d->{type} || $d->{type} ne 'qemu';
2425         $res->{$vmid}->{exists} = 1;
2426     }
2427     return $res;
2428 }
2429
2430 # test if VM uses local resources (to prevent migration)
2431 sub check_local_resources {
2432     my ($conf, $noerr) = @_;
2433
2434     my @loc_res = ();
2435
2436     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2437     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2438
2439     push @loc_res, "ivshmem" if $conf->{ivshmem};
2440
2441     foreach my $k (keys %$conf) {
2442         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2443         # sockets are safe: they will recreated be on the target side post-migrate
2444         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2445         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2446     }
2447
2448     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2449
2450     return \@loc_res;
2451 }
2452
2453 # check if used storages are available on all nodes (use by migrate)
2454 sub check_storage_availability {
2455     my ($storecfg, $conf, $node) = @_;
2456
2457     PVE::QemuConfig->foreach_volume($conf, sub {
2458         my ($ds, $drive) = @_;
2459
2460         my $volid = $drive->{file};
2461         return if !$volid;
2462
2463         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2464         return if !$sid;
2465
2466         # check if storage is available on both nodes
2467         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2468         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2469
2470         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2471
2472         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2473             if !$scfg->{content}->{$vtype};
2474    });
2475 }
2476
2477 # list nodes where all VM images are available (used by has_feature API)
2478 sub shared_nodes {
2479     my ($conf, $storecfg) = @_;
2480
2481     my $nodelist = PVE::Cluster::get_nodelist();
2482     my $nodehash = { map { $_ => 1 } @$nodelist };
2483     my $nodename = nodename();
2484
2485     PVE::QemuConfig->foreach_volume($conf, sub {
2486         my ($ds, $drive) = @_;
2487
2488         my $volid = $drive->{file};
2489         return if !$volid;
2490
2491         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2492         if ($storeid) {
2493             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2494             if ($scfg->{disable}) {
2495                 $nodehash = {};
2496             } elsif (my $avail = $scfg->{nodes}) {
2497                 foreach my $node (keys %$nodehash) {
2498                     delete $nodehash->{$node} if !$avail->{$node};
2499                 }
2500             } elsif (!$scfg->{shared}) {
2501                 foreach my $node (keys %$nodehash) {
2502                     delete $nodehash->{$node} if $node ne $nodename
2503                 }
2504             }
2505         }
2506     });
2507
2508     return $nodehash
2509 }
2510
2511 sub check_local_storage_availability {
2512     my ($conf, $storecfg) = @_;
2513
2514     my $nodelist = PVE::Cluster::get_nodelist();
2515     my $nodehash = { map { $_ => {} } @$nodelist };
2516
2517     PVE::QemuConfig->foreach_volume($conf, sub {
2518         my ($ds, $drive) = @_;
2519
2520         my $volid = $drive->{file};
2521         return if !$volid;
2522
2523         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2524         if ($storeid) {
2525             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2526
2527             if ($scfg->{disable}) {
2528                 foreach my $node (keys %$nodehash) {
2529                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2530                 }
2531             } elsif (my $avail = $scfg->{nodes}) {
2532                 foreach my $node (keys %$nodehash) {
2533                     if (!$avail->{$node}) {
2534                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2535                     }
2536                 }
2537             }
2538         }
2539     });
2540
2541     foreach my $node (values %$nodehash) {
2542         if (my $unavail = $node->{unavailable_storages}) {
2543             $node->{unavailable_storages} = [ sort keys %$unavail ];
2544         }
2545     }
2546
2547     return $nodehash
2548 }
2549
2550 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2551 sub check_running {
2552     my ($vmid, $nocheck, $node) = @_;
2553
2554     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2555     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2556 }
2557
2558 sub vzlist {
2559
2560     my $vzlist = config_list();
2561
2562     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2563
2564     while (defined(my $de = $fd->read)) {
2565         next if $de !~ m/^(\d+)\.pid$/;
2566         my $vmid = $1;
2567         next if !defined($vzlist->{$vmid});
2568         if (my $pid = check_running($vmid)) {
2569             $vzlist->{$vmid}->{pid} = $pid;
2570         }
2571     }
2572
2573     return $vzlist;
2574 }
2575
2576 our $vmstatus_return_properties = {
2577     vmid => get_standard_option('pve-vmid'),
2578     status => {
2579         description => "Qemu process status.",
2580         type => 'string',
2581         enum => ['stopped', 'running'],
2582     },
2583     maxmem => {
2584         description => "Maximum memory in bytes.",
2585         type => 'integer',
2586         optional => 1,
2587         renderer => 'bytes',
2588     },
2589     maxdisk => {
2590         description => "Root disk size in bytes.",
2591         type => 'integer',
2592         optional => 1,
2593         renderer => 'bytes',
2594     },
2595     name => {
2596         description => "VM name.",
2597         type => 'string',
2598         optional => 1,
2599     },
2600     qmpstatus => {
2601         description => "Qemu QMP agent status.",
2602         type => 'string',
2603         optional => 1,
2604     },
2605     pid => {
2606         description => "PID of running qemu process.",
2607         type => 'integer',
2608         optional => 1,
2609     },
2610     uptime => {
2611         description => "Uptime.",
2612         type => 'integer',
2613         optional => 1,
2614         renderer => 'duration',
2615     },
2616     cpus => {
2617         description => "Maximum usable CPUs.",
2618         type => 'number',
2619         optional => 1,
2620     },
2621     lock => {
2622         description => "The current config lock, if any.",
2623         type => 'string',
2624         optional => 1,
2625     },
2626     tags => {
2627         description  => "The current configured tags, if any",
2628         type => 'string',
2629         optional => 1,
2630     },
2631     'running-machine' => {
2632         description => "The currently running machine type (if running).",
2633         type => 'string',
2634         optional => 1,
2635     },
2636     'running-qemu' => {
2637         description => "The currently running QEMU version (if running).",
2638         type => 'string',
2639         optional => 1,
2640     },
2641 };
2642
2643 my $last_proc_pid_stat;
2644
2645 # get VM status information
2646 # This must be fast and should not block ($full == false)
2647 # We only query KVM using QMP if $full == true (this can be slow)
2648 sub vmstatus {
2649     my ($opt_vmid, $full) = @_;
2650
2651     my $res = {};
2652
2653     my $storecfg = PVE::Storage::config();
2654
2655     my $list = vzlist();
2656     my $defaults = load_defaults();
2657
2658     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2659
2660     my $cpucount = $cpuinfo->{cpus} || 1;
2661
2662     foreach my $vmid (keys %$list) {
2663         next if $opt_vmid && ($vmid ne $opt_vmid);
2664
2665         my $conf = PVE::QemuConfig->load_config($vmid);
2666
2667         my $d = { vmid => int($vmid) };
2668         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2669
2670         # fixme: better status?
2671         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2672
2673         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2674         if (defined($size)) {
2675             $d->{disk} = 0; # no info available
2676             $d->{maxdisk} = $size;
2677         } else {
2678             $d->{disk} = 0;
2679             $d->{maxdisk} = 0;
2680         }
2681
2682         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2683             * ($conf->{cores} || $defaults->{cores});
2684         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2685         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2686
2687         $d->{name} = $conf->{name} || "VM $vmid";
2688         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2689             : $defaults->{memory}*(1024*1024);
2690
2691         if ($conf->{balloon}) {
2692             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2693             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2694                 : $defaults->{shares};
2695         }
2696
2697         $d->{uptime} = 0;
2698         $d->{cpu} = 0;
2699         $d->{mem} = 0;
2700
2701         $d->{netout} = 0;
2702         $d->{netin} = 0;
2703
2704         $d->{diskread} = 0;
2705         $d->{diskwrite} = 0;
2706
2707         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2708
2709         $d->{serial} = 1 if conf_has_serial($conf);
2710         $d->{lock} = $conf->{lock} if $conf->{lock};
2711         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2712
2713         $res->{$vmid} = $d;
2714     }
2715
2716     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2717     foreach my $dev (keys %$netdev) {
2718         next if $dev !~ m/^tap([1-9]\d*)i/;
2719         my $vmid = $1;
2720         my $d = $res->{$vmid};
2721         next if !$d;
2722
2723         $d->{netout} += $netdev->{$dev}->{receive};
2724         $d->{netin} += $netdev->{$dev}->{transmit};
2725
2726         if ($full) {
2727             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2728             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2729         }
2730
2731     }
2732
2733     my $ctime = gettimeofday;
2734
2735     foreach my $vmid (keys %$list) {
2736
2737         my $d = $res->{$vmid};
2738         my $pid = $d->{pid};
2739         next if !$pid;
2740
2741         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2742         next if !$pstat; # not running
2743
2744         my $used = $pstat->{utime} + $pstat->{stime};
2745
2746         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2747
2748         if ($pstat->{vsize}) {
2749             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2750         }
2751
2752         my $old = $last_proc_pid_stat->{$pid};
2753         if (!$old) {
2754             $last_proc_pid_stat->{$pid} = {
2755                 time => $ctime,
2756                 used => $used,
2757                 cpu => 0,
2758             };
2759             next;
2760         }
2761
2762         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2763
2764         if ($dtime > 1000) {
2765             my $dutime = $used -  $old->{used};
2766
2767             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2768             $last_proc_pid_stat->{$pid} = {
2769                 time => $ctime,
2770                 used => $used,
2771                 cpu => $d->{cpu},
2772             };
2773         } else {
2774             $d->{cpu} = $old->{cpu};
2775         }
2776     }
2777
2778     return $res if !$full;
2779
2780     my $qmpclient = PVE::QMPClient->new();
2781
2782     my $ballooncb = sub {
2783         my ($vmid, $resp) = @_;
2784
2785         my $info = $resp->{'return'};
2786         return if !$info->{max_mem};
2787
2788         my $d = $res->{$vmid};
2789
2790         # use memory assigned to VM
2791         $d->{maxmem} = $info->{max_mem};
2792         $d->{balloon} = $info->{actual};
2793
2794         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2795             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2796             $d->{freemem} = $info->{free_mem};
2797         }
2798
2799         $d->{ballooninfo} = $info;
2800     };
2801
2802     my $blockstatscb = sub {
2803         my ($vmid, $resp) = @_;
2804         my $data = $resp->{'return'} || [];
2805         my $totalrdbytes = 0;
2806         my $totalwrbytes = 0;
2807
2808         for my $blockstat (@$data) {
2809             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2810             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2811
2812             $blockstat->{device} =~ s/drive-//;
2813             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2814         }
2815         $res->{$vmid}->{diskread} = $totalrdbytes;
2816         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2817     };
2818
2819     my $machinecb = sub {
2820         my ($vmid, $resp) = @_;
2821         my $data = $resp->{'return'} || [];
2822
2823         $res->{$vmid}->{'running-machine'} =
2824             PVE::QemuServer::Machine::current_from_query_machines($data);
2825     };
2826
2827     my $versioncb = sub {
2828         my ($vmid, $resp) = @_;
2829         my $data = $resp->{'return'} // {};
2830         my $version = 'unknown';
2831
2832         if (my $v = $data->{qemu}) {
2833             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2834         }
2835
2836         $res->{$vmid}->{'running-qemu'} = $version;
2837     };
2838
2839     my $statuscb = sub {
2840         my ($vmid, $resp) = @_;
2841
2842         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2843         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2844         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2845         # this fails if ballon driver is not loaded, so this must be
2846         # the last commnand (following command are aborted if this fails).
2847         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2848
2849         my $status = 'unknown';
2850         if (!defined($status = $resp->{'return'}->{status})) {
2851             warn "unable to get VM status\n";
2852             return;
2853         }
2854
2855         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2856     };
2857
2858     foreach my $vmid (keys %$list) {
2859         next if $opt_vmid && ($vmid ne $opt_vmid);
2860         next if !$res->{$vmid}->{pid}; # not running
2861         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2862     }
2863
2864     $qmpclient->queue_execute(undef, 2);
2865
2866     foreach my $vmid (keys %$list) {
2867         next if $opt_vmid && ($vmid ne $opt_vmid);
2868         next if !$res->{$vmid}->{pid}; #not running
2869
2870         # we can't use the $qmpclient since it might have already aborted on
2871         # 'query-balloon', but this might also fail for older versions...
2872         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2873         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2874     }
2875
2876     foreach my $vmid (keys %$list) {
2877         next if $opt_vmid && ($vmid ne $opt_vmid);
2878         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2879     }
2880
2881     return $res;
2882 }
2883
2884 sub conf_has_serial {
2885     my ($conf) = @_;
2886
2887     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2888         if ($conf->{"serial$i"}) {
2889             return 1;
2890         }
2891     }
2892
2893     return 0;
2894 }
2895
2896 sub conf_has_audio {
2897     my ($conf, $id) = @_;
2898
2899     $id //= 0;
2900     my $audio = $conf->{"audio$id"};
2901     return if !defined($audio);
2902
2903     my $audioproperties = parse_property_string($audio_fmt, $audio);
2904     my $audiodriver = $audioproperties->{driver} // 'spice';
2905
2906     return {
2907         dev => $audioproperties->{device},
2908         dev_id => "audiodev$id",
2909         backend => $audiodriver,
2910         backend_id => "$audiodriver-backend${id}",
2911     };
2912 }
2913
2914 sub audio_devs {
2915     my ($audio, $audiopciaddr, $machine_version) = @_;
2916
2917     my $devs = [];
2918
2919     my $id = $audio->{dev_id};
2920     my $audiodev = "";
2921     if (min_version($machine_version, 4, 2)) {
2922         $audiodev = ",audiodev=$audio->{backend_id}";
2923     }
2924
2925     if ($audio->{dev} eq 'AC97') {
2926         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2927     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2928         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2929         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2930         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2931     } else {
2932         die "unkown audio device '$audio->{dev}', implement me!";
2933     }
2934
2935     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2936
2937     return $devs;
2938 }
2939
2940 sub vga_conf_has_spice {
2941     my ($vga) = @_;
2942
2943     my $vgaconf = parse_vga($vga);
2944     my $vgatype = $vgaconf->{type};
2945     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2946
2947     return $1 || 1;
2948 }
2949
2950 sub is_native($) {
2951     my ($arch) = @_;
2952     return get_host_arch() eq $arch;
2953 }
2954
2955 sub get_vm_arch {
2956     my ($conf) = @_;
2957     return $conf->{arch} // get_host_arch();
2958 }
2959
2960 my $default_machines = {
2961     x86_64 => 'pc',
2962     aarch64 => 'virt',
2963 };
2964
2965 sub get_installed_machine_version {
2966     my ($kvmversion) = @_;
2967     $kvmversion = kvm_user_version() if !defined($kvmversion);
2968     $kvmversion =~ m/^(\d+\.\d+)/;
2969     return $1;
2970 }
2971
2972 sub windows_get_pinned_machine_version {
2973     my ($machine, $base_version, $kvmversion) = @_;
2974
2975     my $pin_version = $base_version;
2976     if (!defined($base_version) ||
2977         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2978     ) {
2979         $pin_version = get_installed_machine_version($kvmversion);
2980     }
2981     if (!$machine || $machine eq 'pc') {
2982         $machine = "pc-i440fx-$pin_version";
2983     } elsif ($machine eq 'q35') {
2984         $machine = "pc-q35-$pin_version";
2985     } elsif ($machine eq 'virt') {
2986         $machine = "virt-$pin_version";
2987     } else {
2988         warn "unknown machine type '$machine', not touching that!\n";
2989     }
2990
2991     return $machine;
2992 }
2993
2994 sub get_vm_machine {
2995     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2996
2997     my $machine = $forcemachine || $conf->{machine};
2998
2999     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3000         $kvmversion //= kvm_user_version();
3001         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3002         # layout which confuses windows quite a bit and may result in various regressions..
3003         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3004         if (windows_version($conf->{ostype})) {
3005             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3006         }
3007         $arch //= 'x86_64';
3008         $machine ||= $default_machines->{$arch};
3009         if ($add_pve_version) {
3010             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3011             $machine .= "+pve$pvever";
3012         }
3013     }
3014
3015     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3016         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3017         $machine = $1 if $is_pxe;
3018
3019         # for version-pinned machines that do not include a pve-version (e.g.
3020         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3021         $machine .= '+pve0';
3022
3023         $machine .= '.pxe' if $is_pxe;
3024     }
3025
3026     return $machine;
3027 }
3028
3029 sub get_ovmf_files($) {
3030     my ($arch) = @_;
3031
3032     my $ovmf = $OVMF->{$arch}
3033         or die "no OVMF images known for architecture '$arch'\n";
3034
3035     return @$ovmf;
3036 }
3037
3038 my $Arch2Qemu = {
3039     aarch64 => '/usr/bin/qemu-system-aarch64',
3040     x86_64 => '/usr/bin/qemu-system-x86_64',
3041 };
3042 sub get_command_for_arch($) {
3043     my ($arch) = @_;
3044     return '/usr/bin/kvm' if is_native($arch);
3045
3046     my $cmd = $Arch2Qemu->{$arch}
3047         or die "don't know how to emulate architecture '$arch'\n";
3048     return $cmd;
3049 }
3050
3051 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3052 # to use in a QEMU command line (-cpu element), first array_intersect the result
3053 # of query_supported_ with query_understood_. This is necessary because:
3054 #
3055 # a) query_understood_ returns flags the host cannot use and
3056 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3057 #    flags, but CPU settings - with most of them being flags. Those settings
3058 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3059 #
3060 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3061 # expensive. If you need the value returned from this, you can get it much
3062 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3063 # $accel being 'kvm' or 'tcg'.
3064 #
3065 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3066 # changes, automatically populating pmxcfs.
3067 #
3068 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3069 # since kvm and tcg machines support different flags
3070 #
3071 sub query_supported_cpu_flags {
3072     my ($arch) = @_;
3073
3074     $arch //= get_host_arch();
3075     my $default_machine = $default_machines->{$arch};
3076
3077     my $flags = {};
3078
3079     # FIXME: Once this is merged, the code below should work for ARM as well:
3080     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3081     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3082         $arch eq "aarch64";
3083
3084     my $kvm_supported = defined(kvm_version());
3085     my $qemu_cmd = get_command_for_arch($arch);
3086     my $fakevmid = -1;
3087     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3088
3089     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3090     my $query_supported_run_qemu = sub {
3091         my ($kvm) = @_;
3092
3093         my $flags = {};
3094         my $cmd = [
3095             $qemu_cmd,
3096             '-machine', $default_machine,
3097             '-display', 'none',
3098             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3099             '-mon', 'chardev=qmp,mode=control',
3100             '-pidfile', $pidfile,
3101             '-S', '-daemonize'
3102         ];
3103
3104         if (!$kvm) {
3105             push @$cmd, '-accel', 'tcg';
3106         }
3107
3108         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3109         die "QEMU flag querying VM exited with code " . $rc if $rc;
3110
3111         eval {
3112             my $cmd_result = mon_cmd(
3113                 $fakevmid,
3114                 'query-cpu-model-expansion',
3115                 type => 'full',
3116                 model => { name => 'host' }
3117             );
3118
3119             my $props = $cmd_result->{model}->{props};
3120             foreach my $prop (keys %$props) {
3121                 next if $props->{$prop} ne '1';
3122                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3123                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3124                 # We only keep those with underscores, to match /proc/cpuinfo
3125                 $prop =~ s/\.|-/_/g;
3126                 $flags->{$prop} = 1;
3127             }
3128         };
3129         my $err = $@;
3130
3131         # force stop with 10 sec timeout and 'nocheck'
3132         # always stop, even if QMP failed
3133         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3134
3135         die $err if $err;
3136
3137         return [ sort keys %$flags ];
3138     };
3139
3140     # We need to query QEMU twice, since KVM and TCG have different supported flags
3141     PVE::QemuConfig->lock_config($fakevmid, sub {
3142         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3143         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3144
3145         if ($kvm_supported) {
3146             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3147             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3148         }
3149     });
3150
3151     return $flags;
3152 }
3153
3154 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3155 my $understood_cpu_flag_dir = "/usr/share/kvm";
3156 sub query_understood_cpu_flags {
3157     my $arch = get_host_arch();
3158     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3159
3160     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3161         if ! -e $filepath;
3162
3163     my $raw = file_get_contents($filepath);
3164     $raw =~ s/^\s+|\s+$//g;
3165     my @flags = split(/\s+/, $raw);
3166
3167     return \@flags;
3168 }
3169
3170 sub config_to_command {
3171     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3172         $pbs_backing) = @_;
3173
3174     my $cmd = [];
3175     my $globalFlags = [];
3176     my $machineFlags = [];
3177     my $rtcFlags = [];
3178     my $devices = [];
3179     my $pciaddr = '';
3180     my $bridges = {};
3181     my $ostype = $conf->{ostype};
3182     my $winversion = windows_version($ostype);
3183     my $kvm = $conf->{kvm};
3184     my $nodename = nodename();
3185
3186     my $arch = get_vm_arch($conf);
3187     my $kvm_binary = get_command_for_arch($arch);
3188     my $kvmver = kvm_user_version($kvm_binary);
3189
3190     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3191         $kvmver //= "undefined";
3192         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3193     }
3194
3195     my $add_pve_version = min_version($kvmver, 4, 1);
3196
3197     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3198     my $machine_version = extract_version($machine_type, $kvmver);
3199     $kvm //= 1 if is_native($arch);
3200
3201     $machine_version =~ m/(\d+)\.(\d+)/;
3202     my ($machine_major, $machine_minor) = ($1, $2);
3203
3204     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3205         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3206     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3207         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3208             ." please upgrade node '$nodename'\n"
3209     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3210         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3211         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3212             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3213             ." node '$nodename'\n";
3214     }
3215
3216     # if a specific +pve version is required for a feature, use $version_guard
3217     # instead of min_version to allow machines to be run with the minimum
3218     # required version
3219     my $required_pve_version = 0;
3220     my $version_guard = sub {
3221         my ($major, $minor, $pve) = @_;
3222         return 0 if !min_version($machine_version, $major, $minor, $pve);
3223         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3224         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3225         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3226         return 1;
3227     };
3228
3229     if ($kvm && !defined kvm_version()) {
3230         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3231             ." or enable in BIOS.\n";
3232     }
3233
3234     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3235     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3236     my $use_old_bios_files = undef;
3237     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3238
3239     my $cpuunits = defined($conf->{cpuunits}) ?
3240             $conf->{cpuunits} : $defaults->{cpuunits};
3241
3242     push @$cmd, $kvm_binary;
3243
3244     push @$cmd, '-id', $vmid;
3245
3246     my $vmname = $conf->{name} || "vm$vmid";
3247
3248     push @$cmd, '-name', $vmname;
3249
3250     push @$cmd, '-no-shutdown';
3251
3252     my $use_virtio = 0;
3253
3254     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3255     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3256     push @$cmd, '-mon', "chardev=qmp,mode=control";
3257
3258     if (min_version($machine_version, 2, 12)) {
3259         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3260         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3261     }
3262
3263     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3264
3265     push @$cmd, '-daemonize';
3266
3267     if ($conf->{smbios1}) {
3268         my $smbios_conf = parse_smbios1($conf->{smbios1});
3269         if ($smbios_conf->{base64}) {
3270             # Do not pass base64 flag to qemu
3271             delete $smbios_conf->{base64};
3272             my $smbios_string = "";
3273             foreach my $key (keys %$smbios_conf) {
3274                 my $value;
3275                 if ($key eq "uuid") {
3276                     $value = $smbios_conf->{uuid}
3277                 } else {
3278                     $value = decode_base64($smbios_conf->{$key});
3279                 }
3280                 # qemu accepts any binary data, only commas need escaping by double comma
3281                 $value =~ s/,/,,/g;
3282                 $smbios_string .= "," . $key . "=" . $value if $value;
3283             }
3284             push @$cmd, '-smbios', "type=1" . $smbios_string;
3285         } else {
3286             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3287         }
3288     }
3289
3290     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3291         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3292         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3293
3294         my ($path, $format);
3295         my $read_only_str = '';
3296         if (my $efidisk = $conf->{efidisk0}) {
3297             my $d = parse_drive('efidisk0', $efidisk);
3298             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3299             $format = $d->{format};
3300             if ($storeid) {
3301                 $path = PVE::Storage::path($storecfg, $d->{file});
3302                 if (!defined($format)) {
3303                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3304                     $format = qemu_img_format($scfg, $volname);
3305                 }
3306             } else {
3307                 $path = $d->{file};
3308                 die "efidisk format must be specified\n"
3309                     if !defined($format);
3310             }
3311
3312             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3313         } else {
3314             warn "no efidisk configured! Using temporary efivars disk.\n";
3315             $path = "/tmp/$vmid-ovmf.fd";
3316             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3317             $format = 'raw';
3318         }
3319
3320         my $size_str = "";
3321
3322         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3323             $size_str = ",size=" . (-s $ovmf_vars);
3324         }
3325
3326         # on slower ceph clusters, booting without cache on efidisk can take a while, see #3329
3327         my $cache = $path =~ m/^rbd:/ ? ',cache=writeback' : '';
3328
3329         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3330         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3331     }
3332
3333     # load q35 config
3334     if ($q35) {
3335         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3336         if (min_version($machine_version, 4, 0)) {
3337             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3338         } else {
3339             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3340         }
3341     }
3342
3343     if ($conf->{vmgenid}) {
3344         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3345     }
3346
3347     # add usb controllers
3348     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3349         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3350     push @$devices, @usbcontrollers if @usbcontrollers;
3351     my $vga = parse_vga($conf->{vga});
3352
3353     my $qxlnum = vga_conf_has_spice($conf->{vga});
3354     $vga->{type} = 'qxl' if $qxlnum;
3355
3356     if (!$vga->{type}) {
3357         if ($arch eq 'aarch64') {
3358             $vga->{type} = 'virtio';
3359         } elsif (min_version($machine_version, 2, 9)) {
3360             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3361         } else {
3362             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3363         }
3364     }
3365
3366     # enable absolute mouse coordinates (needed by vnc)
3367     my $tablet;
3368     if (defined($conf->{tablet})) {
3369         $tablet = $conf->{tablet};
3370     } else {
3371         $tablet = $defaults->{tablet};
3372         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3373         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3374     }
3375
3376     if ($tablet) {
3377         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3378         my $kbd = print_keyboarddevice_full($conf, $arch);
3379         push @$devices, '-device', $kbd if defined($kbd);
3380     }
3381
3382     my $bootorder = device_bootorder($conf);
3383
3384     # host pci device passthrough
3385     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3386         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3387
3388     # usb devices
3389     my $usb_dev_features = {};
3390     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3391
3392     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3393         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3394     push @$devices, @usbdevices if @usbdevices;
3395
3396     # serial devices
3397     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3398         if (my $path = $conf->{"serial$i"}) {
3399             if ($path eq 'socket') {
3400                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3401                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3402                 # On aarch64, serial0 is the UART device. Qemu only allows
3403                 # connecting UART devices via the '-serial' command line, as
3404                 # the device has a fixed slot on the hardware...
3405                 if ($arch eq 'aarch64' && $i == 0) {
3406                     push @$devices, '-serial', "chardev:serial$i";
3407                 } else {
3408                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3409                 }
3410             } else {
3411                 die "no such serial device\n" if ! -c $path;
3412                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3413                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3414             }
3415         }
3416     }
3417
3418     # parallel devices
3419     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3420         if (my $path = $conf->{"parallel$i"}) {
3421             die "no such parallel device\n" if ! -c $path;
3422             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3423             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3424             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3425         }
3426     }
3427
3428     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3429         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3430         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3431         push @$devices, @$audio_devs;
3432     }
3433
3434     my $sockets = 1;
3435     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3436     $sockets = $conf->{sockets} if  $conf->{sockets};
3437
3438     my $cores = $conf->{cores} || 1;
3439
3440     my $maxcpus = $sockets * $cores;
3441
3442     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3443
3444     my $allowed_vcpus = $cpuinfo->{cpus};
3445
3446     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3447         if ($allowed_vcpus < $maxcpus);
3448
3449     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3450
3451         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3452         for (my $i = 2; $i <= $vcpus; $i++)  {
3453             my $cpustr = print_cpu_device($conf,$i);
3454             push @$cmd, '-device', $cpustr;
3455         }
3456
3457     } else {
3458
3459         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3460     }
3461     push @$cmd, '-nodefaults';
3462
3463     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3464
3465     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3466
3467     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3468
3469     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3470         push @$devices, '-device', print_vga_device(
3471             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3472         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3473         push @$cmd,  '-vnc', "unix:$socket,password=on";
3474     } else {
3475         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3476         push @$cmd, '-nographic';
3477     }
3478
3479     # time drift fix
3480     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3481     my $useLocaltime = $conf->{localtime};
3482
3483     if ($winversion >= 5) { # windows
3484         $useLocaltime = 1 if !defined($conf->{localtime});
3485
3486         # use time drift fix when acpi is enabled
3487         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3488             $tdf = 1 if !defined($conf->{tdf});
3489         }
3490     }
3491
3492     if ($winversion >= 6) {
3493         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3494         push @$cmd, '-no-hpet';
3495     }
3496
3497     push @$rtcFlags, 'driftfix=slew' if $tdf;
3498
3499     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3500         push @$rtcFlags, "base=$conf->{startdate}";
3501     } elsif ($useLocaltime) {
3502         push @$rtcFlags, 'base=localtime';
3503     }
3504
3505     if ($forcecpu) {
3506         push @$cmd, '-cpu', $forcecpu;
3507     } else {
3508         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3509     }
3510
3511     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3512
3513     push @$cmd, '-S' if $conf->{freeze};
3514
3515     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3516
3517     my $guest_agent = parse_guest_agent($conf);
3518
3519     if ($guest_agent->{enabled}) {
3520         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3521         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3522
3523         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3524             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3525             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3526             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3527         } elsif ($guest_agent->{type} eq 'isa') {
3528             push @$devices, '-device', "isa-serial,chardev=qga0";
3529         }
3530     }
3531
3532     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3533     if ($rng && $version_guard->(4, 1, 2)) {
3534         check_rng_source($rng->{source});
3535
3536         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3537         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3538         my $limiter_str = "";
3539         if ($max_bytes) {
3540             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3541         }
3542
3543         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3544         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3545         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3546     }
3547
3548     my $spice_port;
3549
3550     if ($qxlnum) {
3551         if ($qxlnum > 1) {
3552             if ($winversion){
3553                 for (my $i = 1; $i < $qxlnum; $i++){
3554                     push @$devices, '-device', print_vga_device(
3555                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3556                 }
3557             } else {
3558                 # assume other OS works like Linux
3559                 my ($ram, $vram) = ("134217728", "67108864");
3560                 if ($vga->{memory}) {
3561                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3562                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3563                 }
3564                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3565                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3566             }
3567         }
3568
3569         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3570
3571         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3572         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3573         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3574
3575         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3576         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3577         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3578
3579         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3580         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3581
3582         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3583         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3584         if ($spice_enhancement->{foldersharing}) {
3585             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3586             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3587         }
3588
3589         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3590         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3591             if $spice_enhancement->{videostreaming};
3592
3593         push @$devices, '-spice', "$spice_opts";
3594     }
3595
3596     # enable balloon by default, unless explicitly disabled
3597     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3598         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3599         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3600     }
3601
3602     if ($conf->{watchdog}) {
3603         my $wdopts = parse_watchdog($conf->{watchdog});
3604         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3605         my $watchdog = $wdopts->{model} || 'i6300esb';
3606         push @$devices, '-device', "$watchdog$pciaddr";
3607         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3608     }
3609
3610     my $vollist = [];
3611     my $scsicontroller = {};
3612     my $ahcicontroller = {};
3613     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3614
3615     # Add iscsi initiator name if available
3616     if (my $initiator = get_initiator_name()) {
3617         push @$devices, '-iscsi', "initiator-name=$initiator";
3618     }
3619
3620     PVE::QemuConfig->foreach_volume($conf, sub {
3621         my ($ds, $drive) = @_;
3622
3623         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3624             check_volume_storage_type($storecfg, $drive->{file});
3625             push @$vollist, $drive->{file};
3626         }
3627
3628         # ignore efidisk here, already added in bios/fw handling code above
3629         return if $drive->{interface} eq 'efidisk';
3630
3631         $use_virtio = 1 if $ds =~ m/^virtio/;
3632
3633         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3634
3635         if ($drive->{interface} eq 'virtio'){
3636            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3637         }
3638
3639         if ($drive->{interface} eq 'scsi') {
3640
3641             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3642
3643             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3644                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3645
3646             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3647             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3648
3649             my $iothread = '';
3650             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3651                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3652                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3653             } elsif ($drive->{iothread}) {
3654                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3655             }
3656
3657             my $queues = '';
3658             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3659                 $queues = ",num_queues=$drive->{queues}";
3660             }
3661
3662             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3663                 if !$scsicontroller->{$controller};
3664             $scsicontroller->{$controller}=1;
3665         }
3666
3667         if ($drive->{interface} eq 'sata') {
3668             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3669             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3670             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3671                 if !$ahcicontroller->{$controller};
3672             $ahcicontroller->{$controller}=1;
3673         }
3674
3675         my $pbs_conf = $pbs_backing->{$ds};
3676         my $pbs_name = undef;
3677         if ($pbs_conf) {
3678             $pbs_name = "drive-$ds-pbs";
3679             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3680         }
3681
3682         my $drive_cmd = print_drive_commandline_full(
3683             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3684
3685         # extra protection for templates, but SATA and IDE don't support it..
3686         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3687
3688         push @$devices, '-drive',$drive_cmd;
3689         push @$devices, '-device', print_drivedevice_full(
3690             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3691     });
3692
3693     for (my $i = 0; $i < $MAX_NETS; $i++) {
3694         my $netname = "net$i";
3695
3696         next if !$conf->{$netname};
3697         my $d = parse_net($conf->{$netname});
3698         next if !$d;
3699
3700         $use_virtio = 1 if $d->{model} eq 'virtio';
3701
3702         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3703
3704         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3705         push @$devices, '-netdev', $netdevfull;
3706
3707         my $netdevicefull = print_netdevice_full(
3708             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3709
3710         push @$devices, '-device', $netdevicefull;
3711     }
3712
3713     if ($conf->{ivshmem}) {
3714         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3715
3716         my $bus;
3717         if ($q35) {
3718             $bus = print_pcie_addr("ivshmem");
3719         } else {
3720             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3721         }
3722
3723         my $ivshmem_name = $ivshmem->{name} // $vmid;
3724         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3725
3726         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3727         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3728             .",size=$ivshmem->{size}M";
3729     }
3730
3731     # pci.4 is nested in pci.1
3732     $bridges->{1} = 1 if $bridges->{4};
3733
3734     if (!$q35) {
3735         # add pci bridges
3736         if (min_version($machine_version, 2, 3)) {
3737            $bridges->{1} = 1;
3738            $bridges->{2} = 1;
3739         }
3740
3741         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3742
3743     }
3744
3745     for my $k (sort {$b cmp $a} keys %$bridges) {
3746         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3747
3748         my $k_name = $k;
3749         if ($k == 2 && $legacy_igd) {
3750             $k_name = "$k-igd";
3751         }
3752         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3753
3754         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3755         if ($q35) {
3756             # add after -readconfig pve-q35.cfg
3757             splice @$devices, 2, 0, '-device', $devstr;
3758         } else {
3759             unshift @$devices, '-device', $devstr if $k > 0;
3760         }
3761     }
3762
3763     if (!$kvm) {
3764         push @$machineFlags, 'accel=tcg';
3765     }
3766
3767     my $machine_type_min = $machine_type;
3768     if ($add_pve_version) {
3769         $machine_type_min =~ s/\+pve\d+$//;
3770         $machine_type_min .= "+pve$required_pve_version";
3771     }
3772     push @$machineFlags, "type=${machine_type_min}";
3773
3774     push @$cmd, @$devices;
3775     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3776     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3777     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3778
3779     if (my $vmstate = $conf->{vmstate}) {
3780         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3781         push @$vollist, $vmstate;
3782         push @$cmd, '-loadstate', $statepath;
3783         print "activating and using '$vmstate' as vmstate\n";
3784     }
3785
3786     if (PVE::QemuConfig->is_template($conf)) {
3787         # needed to workaround base volumes being read-only
3788         push @$cmd, '-snapshot';
3789     }
3790
3791     # add custom args
3792     if ($conf->{args}) {
3793         my $aa = PVE::Tools::split_args($conf->{args});
3794         push @$cmd, @$aa;
3795     }
3796
3797     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3798 }
3799
3800 sub check_rng_source {
3801     my ($source) = @_;
3802
3803     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3804     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3805         if ! -e $source;
3806
3807     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3808     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3809         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3810         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3811         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3812             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3813             ." to the host.\n";
3814     }
3815 }
3816
3817 sub spice_port {
3818     my ($vmid) = @_;
3819
3820     my $res = mon_cmd($vmid, 'query-spice');
3821
3822     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3823 }
3824
3825 sub vm_devices_list {
3826     my ($vmid) = @_;
3827
3828     my $res = mon_cmd($vmid, 'query-pci');
3829     my $devices_to_check = [];
3830     my $devices = {};
3831     foreach my $pcibus (@$res) {
3832         push @$devices_to_check, @{$pcibus->{devices}},
3833     }
3834
3835     while (@$devices_to_check) {
3836         my $to_check = [];
3837         for my $d (@$devices_to_check) {
3838             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3839             next if !$d->{'pci_bridge'};
3840
3841             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3842             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3843         }
3844         $devices_to_check = $to_check;
3845     }
3846
3847     my $resblock = mon_cmd($vmid, 'query-block');
3848     foreach my $block (@$resblock) {
3849         if($block->{device} =~ m/^drive-(\S+)/){
3850                 $devices->{$1} = 1;
3851         }
3852     }
3853
3854     my $resmice = mon_cmd($vmid, 'query-mice');
3855     foreach my $mice (@$resmice) {
3856         if ($mice->{name} eq 'QEMU HID Tablet') {
3857             $devices->{tablet} = 1;
3858             last;
3859         }
3860     }
3861
3862     # for usb devices there is no query-usb
3863     # but we can iterate over the entries in
3864     # qom-list path=/machine/peripheral
3865     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3866     foreach my $per (@$resperipheral) {
3867         if ($per->{name} =~ m/^usb\d+$/) {
3868             $devices->{$per->{name}} = 1;
3869         }
3870     }
3871
3872     return $devices;
3873 }
3874
3875 sub vm_deviceplug {
3876     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3877
3878     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3879
3880     my $devices_list = vm_devices_list($vmid);
3881     return 1 if defined($devices_list->{$deviceid});
3882
3883     # add PCI bridge if we need it for the device
3884     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3885
3886     if ($deviceid eq 'tablet') {
3887
3888         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3889
3890     } elsif ($deviceid eq 'keyboard') {
3891
3892         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3893
3894     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3895
3896         die "usb hotplug currently not reliable\n";
3897         # since we can't reliably hot unplug all added usb devices and usb
3898         # passthrough breaks live migration we disable usb hotplugging for now
3899         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3900
3901     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3902
3903         qemu_iothread_add($vmid, $deviceid, $device);
3904
3905         qemu_driveadd($storecfg, $vmid, $device);
3906         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3907
3908         qemu_deviceadd($vmid, $devicefull);
3909         eval { qemu_deviceaddverify($vmid, $deviceid); };
3910         if (my $err = $@) {
3911             eval { qemu_drivedel($vmid, $deviceid); };
3912             warn $@ if $@;
3913             die $err;
3914         }
3915
3916     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3917
3918
3919         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3920         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3921         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3922
3923         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3924
3925         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3926             qemu_iothread_add($vmid, $deviceid, $device);
3927             $devicefull .= ",iothread=iothread-$deviceid";
3928         }
3929
3930         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3931             $devicefull .= ",num_queues=$device->{queues}";
3932         }
3933
3934         qemu_deviceadd($vmid, $devicefull);
3935         qemu_deviceaddverify($vmid, $deviceid);
3936
3937     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3938
3939         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3940         qemu_driveadd($storecfg, $vmid, $device);
3941
3942         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3943         eval { qemu_deviceadd($vmid, $devicefull); };
3944         if (my $err = $@) {
3945             eval { qemu_drivedel($vmid, $deviceid); };
3946             warn $@ if $@;
3947             die $err;
3948         }
3949
3950     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3951
3952         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3953
3954         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3955         my $use_old_bios_files = undef;
3956         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3957
3958         my $netdevicefull = print_netdevice_full(
3959             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3960         qemu_deviceadd($vmid, $netdevicefull);
3961         eval {
3962             qemu_deviceaddverify($vmid, $deviceid);
3963             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3964         };
3965         if (my $err = $@) {
3966             eval { qemu_netdevdel($vmid, $deviceid); };
3967             warn $@ if $@;
3968             die $err;
3969         }
3970
3971     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3972
3973         my $bridgeid = $2;
3974         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3975         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3976
3977         qemu_deviceadd($vmid, $devicefull);
3978         qemu_deviceaddverify($vmid, $deviceid);
3979
3980     } else {
3981         die "can't hotplug device '$deviceid'\n";
3982     }
3983
3984     return 1;
3985 }
3986
3987 # fixme: this should raise exceptions on error!
3988 sub vm_deviceunplug {
3989     my ($vmid, $conf, $deviceid) = @_;
3990
3991     my $devices_list = vm_devices_list($vmid);
3992     return 1 if !defined($devices_list->{$deviceid});
3993
3994     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3995     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3996
3997     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3998
3999         qemu_devicedel($vmid, $deviceid);
4000
4001     } elsif ($deviceid =~ m/^usb\d+$/) {
4002
4003         die "usb hotplug currently not reliable\n";
4004         # when unplugging usb devices this way, there may be remaining usb
4005         # controllers/hubs so we disable it for now
4006         #qemu_devicedel($vmid, $deviceid);
4007         #qemu_devicedelverify($vmid, $deviceid);
4008
4009     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4010
4011         qemu_devicedel($vmid, $deviceid);
4012         qemu_devicedelverify($vmid, $deviceid);
4013         qemu_drivedel($vmid, $deviceid);
4014         qemu_iothread_del($conf, $vmid, $deviceid);
4015
4016     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4017
4018         qemu_devicedel($vmid, $deviceid);
4019         qemu_devicedelverify($vmid, $deviceid);
4020         qemu_iothread_del($conf, $vmid, $deviceid);
4021
4022     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4023
4024         qemu_devicedel($vmid, $deviceid);
4025         qemu_drivedel($vmid, $deviceid);
4026         qemu_deletescsihw($conf, $vmid, $deviceid);
4027
4028     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4029
4030         qemu_devicedel($vmid, $deviceid);
4031         qemu_devicedelverify($vmid, $deviceid);
4032         qemu_netdevdel($vmid, $deviceid);
4033
4034     } else {
4035         die "can't unplug device '$deviceid'\n";
4036     }
4037
4038     return 1;
4039 }
4040
4041 sub qemu_deviceadd {
4042     my ($vmid, $devicefull) = @_;
4043
4044     $devicefull = "driver=".$devicefull;
4045     my %options =  split(/[=,]/, $devicefull);
4046
4047     mon_cmd($vmid, "device_add" , %options);
4048 }
4049
4050 sub qemu_devicedel {
4051     my ($vmid, $deviceid) = @_;
4052
4053     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4054 }
4055
4056 sub qemu_iothread_add {
4057     my($vmid, $deviceid, $device) = @_;
4058
4059     if ($device->{iothread}) {
4060         my $iothreads = vm_iothreads_list($vmid);
4061         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4062     }
4063 }
4064
4065 sub qemu_iothread_del {
4066     my($conf, $vmid, $deviceid) = @_;
4067
4068     my $confid = $deviceid;
4069     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4070         $confid = 'scsi' . $1;
4071     }
4072     my $device = parse_drive($confid, $conf->{$confid});
4073     if ($device->{iothread}) {
4074         my $iothreads = vm_iothreads_list($vmid);
4075         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4076     }
4077 }
4078
4079 sub qemu_objectadd {
4080     my($vmid, $objectid, $qomtype) = @_;
4081
4082     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4083
4084     return 1;
4085 }
4086
4087 sub qemu_objectdel {
4088     my($vmid, $objectid) = @_;
4089
4090     mon_cmd($vmid, "object-del", id => $objectid);
4091
4092     return 1;
4093 }
4094
4095 sub qemu_driveadd {
4096     my ($storecfg, $vmid, $device) = @_;
4097
4098     my $kvmver = get_running_qemu_version($vmid);
4099     my $io_uring = min_version($kvmver, 6, 0);
4100     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4101     $drive =~ s/\\/\\\\/g;
4102     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4103
4104     # If the command succeeds qemu prints: "OK"
4105     return 1 if $ret =~ m/OK/s;
4106
4107     die "adding drive failed: $ret\n";
4108 }
4109
4110 sub qemu_drivedel {
4111     my($vmid, $deviceid) = @_;
4112
4113     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4114     $ret =~ s/^\s+//;
4115
4116     return 1 if $ret eq "";
4117
4118     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4119     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4120
4121     die "deleting drive $deviceid failed : $ret\n";
4122 }
4123
4124 sub qemu_deviceaddverify {
4125     my ($vmid, $deviceid) = @_;
4126
4127     for (my $i = 0; $i <= 5; $i++) {
4128          my $devices_list = vm_devices_list($vmid);
4129          return 1 if defined($devices_list->{$deviceid});
4130          sleep 1;
4131     }
4132
4133     die "error on hotplug device '$deviceid'\n";
4134 }
4135
4136
4137 sub qemu_devicedelverify {
4138     my ($vmid, $deviceid) = @_;
4139
4140     # need to verify that the device is correctly removed as device_del
4141     # is async and empty return is not reliable
4142
4143     for (my $i = 0; $i <= 5; $i++) {
4144          my $devices_list = vm_devices_list($vmid);
4145          return 1 if !defined($devices_list->{$deviceid});
4146          sleep 1;
4147     }
4148
4149     die "error on hot-unplugging device '$deviceid'\n";
4150 }
4151
4152 sub qemu_findorcreatescsihw {
4153     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4154
4155     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4156
4157     my $scsihwid="$controller_prefix$controller";
4158     my $devices_list = vm_devices_list($vmid);
4159
4160     if(!defined($devices_list->{$scsihwid})) {
4161         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4162     }
4163
4164     return 1;
4165 }
4166
4167 sub qemu_deletescsihw {
4168     my ($conf, $vmid, $opt) = @_;
4169
4170     my $device = parse_drive($opt, $conf->{$opt});
4171
4172     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4173         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4174         return 1;
4175     }
4176
4177     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4178
4179     my $devices_list = vm_devices_list($vmid);
4180     foreach my $opt (keys %{$devices_list}) {
4181         if (is_valid_drivename($opt)) {
4182             my $drive = parse_drive($opt, $conf->{$opt});
4183             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4184                 return 1;
4185             }
4186         }
4187     }
4188
4189     my $scsihwid="scsihw$controller";
4190
4191     vm_deviceunplug($vmid, $conf, $scsihwid);
4192
4193     return 1;
4194 }
4195
4196 sub qemu_add_pci_bridge {
4197     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4198
4199     my $bridges = {};
4200
4201     my $bridgeid;
4202
4203     print_pci_addr($device, $bridges, $arch, $machine_type);
4204
4205     while (my ($k, $v) = each %$bridges) {
4206         $bridgeid = $k;
4207     }
4208     return 1 if !defined($bridgeid) || $bridgeid < 1;
4209
4210     my $bridge = "pci.$bridgeid";
4211     my $devices_list = vm_devices_list($vmid);
4212
4213     if (!defined($devices_list->{$bridge})) {
4214         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4215     }
4216
4217     return 1;
4218 }
4219
4220 sub qemu_set_link_status {
4221     my ($vmid, $device, $up) = @_;
4222
4223     mon_cmd($vmid, "set_link", name => $device,
4224                up => $up ? JSON::true : JSON::false);
4225 }
4226
4227 sub qemu_netdevadd {
4228     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4229
4230     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4231     my %options =  split(/[=,]/, $netdev);
4232
4233     if (defined(my $vhost = $options{vhost})) {
4234         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4235     }
4236
4237     if (defined(my $queues = $options{queues})) {
4238         $options{queues} = $queues + 0;
4239     }
4240
4241     mon_cmd($vmid, "netdev_add",  %options);
4242     return 1;
4243 }
4244
4245 sub qemu_netdevdel {
4246     my ($vmid, $deviceid) = @_;
4247
4248     mon_cmd($vmid, "netdev_del", id => $deviceid);
4249 }
4250
4251 sub qemu_usb_hotplug {
4252     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4253
4254     return if !$device;
4255
4256     # remove the old one first
4257     vm_deviceunplug($vmid, $conf, $deviceid);
4258
4259     # check if xhci controller is necessary and available
4260     if ($device->{usb3}) {
4261
4262         my $devicelist = vm_devices_list($vmid);
4263
4264         if (!$devicelist->{xhci}) {
4265             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4266             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4267         }
4268     }
4269     my $d = parse_usb_device($device->{host});
4270     $d->{usb3} = $device->{usb3};
4271
4272     # add the new one
4273     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4274 }
4275
4276 sub qemu_cpu_hotplug {
4277     my ($vmid, $conf, $vcpus) = @_;
4278
4279     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4280
4281     my $sockets = 1;
4282     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4283     $sockets = $conf->{sockets} if  $conf->{sockets};
4284     my $cores = $conf->{cores} || 1;
4285     my $maxcpus = $sockets * $cores;
4286
4287     $vcpus = $maxcpus if !$vcpus;
4288
4289     die "you can't add more vcpus than maxcpus\n"
4290         if $vcpus > $maxcpus;
4291
4292     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4293
4294     if ($vcpus < $currentvcpus) {
4295
4296         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4297
4298             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4299                 qemu_devicedel($vmid, "cpu$i");
4300                 my $retry = 0;
4301                 my $currentrunningvcpus = undef;
4302                 while (1) {
4303                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4304                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4305                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4306                     $retry++;
4307                     sleep 1;
4308                 }
4309                 #update conf after each succesfull cpu unplug
4310                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4311                 PVE::QemuConfig->write_config($vmid, $conf);
4312             }
4313         } else {
4314             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4315         }
4316
4317         return;
4318     }
4319
4320     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4321     die "vcpus in running vm does not match its configuration\n"
4322         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4323
4324     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4325
4326         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4327             my $cpustr = print_cpu_device($conf, $i);
4328             qemu_deviceadd($vmid, $cpustr);
4329
4330             my $retry = 0;
4331             my $currentrunningvcpus = undef;
4332             while (1) {
4333                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4334                 last if scalar(@{$currentrunningvcpus}) == $i;
4335                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4336                 sleep 1;
4337                 $retry++;
4338             }
4339             #update conf after each succesfull cpu hotplug
4340             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4341             PVE::QemuConfig->write_config($vmid, $conf);
4342         }
4343     } else {
4344
4345         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4346             mon_cmd($vmid, "cpu-add", id => int($i));
4347         }
4348     }
4349 }
4350
4351 sub qemu_block_set_io_throttle {
4352     my ($vmid, $deviceid,
4353         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4354         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4355         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4356         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4357
4358     return if !check_running($vmid) ;
4359
4360     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4361         bps => int($bps),
4362         bps_rd => int($bps_rd),
4363         bps_wr => int($bps_wr),
4364         iops => int($iops),
4365         iops_rd => int($iops_rd),
4366         iops_wr => int($iops_wr),
4367         bps_max => int($bps_max),
4368         bps_rd_max => int($bps_rd_max),
4369         bps_wr_max => int($bps_wr_max),
4370         iops_max => int($iops_max),
4371         iops_rd_max => int($iops_rd_max),
4372         iops_wr_max => int($iops_wr_max),
4373         bps_max_length => int($bps_max_length),
4374         bps_rd_max_length => int($bps_rd_max_length),
4375         bps_wr_max_length => int($bps_wr_max_length),
4376         iops_max_length => int($iops_max_length),
4377         iops_rd_max_length => int($iops_rd_max_length),
4378         iops_wr_max_length => int($iops_wr_max_length),
4379     );
4380
4381 }
4382
4383 sub qemu_block_resize {
4384     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4385
4386     my $running = check_running($vmid);
4387
4388     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4389
4390     return if !$running;
4391
4392     my $padding = (1024 - $size % 1024) % 1024;
4393     $size = $size + $padding;
4394
4395     mon_cmd(
4396         $vmid,
4397         "block_resize",
4398         device => $deviceid,
4399         size => int($size),
4400         timeout => 60,
4401     );
4402 }
4403
4404 sub qemu_volume_snapshot {
4405     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4406
4407     my $running = check_running($vmid);
4408
4409     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4410         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4411     } else {
4412         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4413     }
4414 }
4415
4416 sub qemu_volume_snapshot_delete {
4417     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4418
4419     my $running = check_running($vmid);
4420
4421     if($running) {
4422
4423         $running = undef;
4424         my $conf = PVE::QemuConfig->load_config($vmid);
4425         PVE::QemuConfig->foreach_volume($conf, sub {
4426             my ($ds, $drive) = @_;
4427             $running = 1 if $drive->{file} eq $volid;
4428         });
4429     }
4430
4431     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4432         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4433     } else {
4434         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4435     }
4436 }
4437
4438 sub set_migration_caps {
4439     my ($vmid, $savevm) = @_;
4440
4441     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4442
4443     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4444     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4445
4446     my $cap_ref = [];
4447
4448     my $enabled_cap = {
4449         "auto-converge" => 1,
4450         "xbzrle" => 1,
4451         "x-rdma-pin-all" => 0,
4452         "zero-blocks" => 0,
4453         "compress" => 0,
4454         "dirty-bitmaps" => $dirty_bitmaps,
4455     };
4456
4457     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4458
4459     for my $supported_capability (@$supported_capabilities) {
4460         push @$cap_ref, {
4461             capability => $supported_capability->{capability},
4462             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4463         };
4464     }
4465
4466     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4467 }
4468
4469 sub foreach_volid {
4470     my ($conf, $func, @param) = @_;
4471
4472     my $volhash = {};
4473
4474     my $test_volid = sub {
4475         my ($key, $drive, $snapname) = @_;
4476
4477         my $volid = $drive->{file};
4478         return if !$volid;
4479
4480         $volhash->{$volid}->{cdrom} //= 1;
4481         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4482
4483         my $replicate = $drive->{replicate} // 1;
4484         $volhash->{$volid}->{replicate} //= 0;
4485         $volhash->{$volid}->{replicate} = 1 if $replicate;
4486
4487         $volhash->{$volid}->{shared} //= 0;
4488         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4489
4490         $volhash->{$volid}->{referenced_in_config} //= 0;
4491         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4492
4493         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4494             if defined($snapname);
4495
4496         my $size = $drive->{size};
4497         $volhash->{$volid}->{size} //= $size if $size;
4498
4499         $volhash->{$volid}->{is_vmstate} //= 0;
4500         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4501
4502         $volhash->{$volid}->{is_unused} //= 0;
4503         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4504
4505         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4506     };
4507
4508     my $include_opts = {
4509         extra_keys => ['vmstate'],
4510         include_unused => 1,
4511     };
4512
4513     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4514     foreach my $snapname (keys %{$conf->{snapshots}}) {
4515         my $snap = $conf->{snapshots}->{$snapname};
4516         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4517     }
4518
4519     foreach my $volid (keys %$volhash) {
4520         &$func($volid, $volhash->{$volid}, @param);
4521     }
4522 }
4523
4524 my $fast_plug_option = {
4525     'lock' => 1,
4526     'name' => 1,
4527     'onboot' => 1,
4528     'shares' => 1,
4529     'startup' => 1,
4530     'description' => 1,
4531     'protection' => 1,
4532     'vmstatestorage' => 1,
4533     'hookscript' => 1,
4534     'tags' => 1,
4535 };
4536
4537 # hotplug changes in [PENDING]
4538 # $selection hash can be used to only apply specified options, for
4539 # example: { cores => 1 } (only apply changed 'cores')
4540 # $errors ref is used to return error messages
4541 sub vmconfig_hotplug_pending {
4542     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4543
4544     my $defaults = load_defaults();
4545     my $arch = get_vm_arch($conf);
4546     my $machine_type = get_vm_machine($conf, undef, $arch);
4547
4548     # commit values which do not have any impact on running VM first
4549     # Note: those option cannot raise errors, we we do not care about
4550     # $selection and always apply them.
4551
4552     my $add_error = sub {
4553         my ($opt, $msg) = @_;
4554         $errors->{$opt} = "hotplug problem - $msg";
4555     };
4556
4557     my $changes = 0;
4558     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4559         if ($fast_plug_option->{$opt}) {
4560             $conf->{$opt} = $conf->{pending}->{$opt};
4561             delete $conf->{pending}->{$opt};
4562             $changes = 1;
4563         }
4564     }
4565
4566     if ($changes) {
4567         PVE::QemuConfig->write_config($vmid, $conf);
4568     }
4569
4570     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4571
4572     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4573     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4574     foreach my $opt (sort keys %$pending_delete_hash) {
4575         next if $selection && !$selection->{$opt};
4576         my $force = $pending_delete_hash->{$opt}->{force};
4577         eval {
4578             if ($opt eq 'hotplug') {
4579                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4580             } elsif ($opt eq 'tablet') {
4581                 die "skip\n" if !$hotplug_features->{usb};
4582                 if ($defaults->{tablet}) {
4583                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4584                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4585                         if $arch eq 'aarch64';
4586                 } else {
4587                     vm_deviceunplug($vmid, $conf, 'tablet');
4588                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4589                 }
4590             } elsif ($opt =~ m/^usb\d+/) {
4591                 die "skip\n";
4592                 # since we cannot reliably hot unplug usb devices we are disabling it
4593                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4594                 #vm_deviceunplug($vmid, $conf, $opt);
4595             } elsif ($opt eq 'vcpus') {
4596                 die "skip\n" if !$hotplug_features->{cpu};
4597                 qemu_cpu_hotplug($vmid, $conf, undef);
4598             } elsif ($opt eq 'balloon') {
4599                 # enable balloon device is not hotpluggable
4600                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4601                 # here we reset the ballooning value to memory
4602                 my $balloon = $conf->{memory} || $defaults->{memory};
4603                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4604             } elsif ($fast_plug_option->{$opt}) {
4605                 # do nothing
4606             } elsif ($opt =~ m/^net(\d+)$/) {
4607                 die "skip\n" if !$hotplug_features->{network};
4608                 vm_deviceunplug($vmid, $conf, $opt);
4609             } elsif (is_valid_drivename($opt)) {
4610                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4611                 vm_deviceunplug($vmid, $conf, $opt);
4612                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4613             } elsif ($opt =~ m/^memory$/) {
4614                 die "skip\n" if !$hotplug_features->{memory};
4615                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4616             } elsif ($opt eq 'cpuunits') {
4617                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4618             } elsif ($opt eq 'cpulimit') {
4619                 $cgroup->change_cpu_quota(-1, 100000);
4620             } else {
4621                 die "skip\n";
4622             }
4623         };
4624         if (my $err = $@) {
4625             &$add_error($opt, $err) if $err ne "skip\n";
4626         } else {
4627             delete $conf->{$opt};
4628             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4629         }
4630     }
4631
4632     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4633     $apply_pending_cloudinit = sub {
4634         return if $apply_pending_cloudinit_done; # once is enough
4635         $apply_pending_cloudinit_done = 1; # once is enough
4636
4637         my ($key, $value) = @_;
4638
4639         my @cloudinit_opts = keys %$confdesc_cloudinit;
4640         foreach my $opt (keys %{$conf->{pending}}) {
4641             next if !grep { $_ eq $opt } @cloudinit_opts;
4642             $conf->{$opt} = delete $conf->{pending}->{$opt};
4643         }
4644
4645         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4646         foreach my $opt (sort keys %$pending_delete_hash) {
4647             next if !grep { $_ eq $opt } @cloudinit_opts;
4648             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4649             delete $conf->{$opt};
4650         }
4651
4652         my $new_conf = { %$conf };
4653         $new_conf->{$key} = $value;
4654         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4655     };
4656
4657     foreach my $opt (keys %{$conf->{pending}}) {
4658         next if $selection && !$selection->{$opt};
4659         my $value = $conf->{pending}->{$opt};
4660         eval {
4661             if ($opt eq 'hotplug') {
4662                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4663             } elsif ($opt eq 'tablet') {
4664                 die "skip\n" if !$hotplug_features->{usb};
4665                 if ($value == 1) {
4666                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4667                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4668                         if $arch eq 'aarch64';
4669                 } elsif ($value == 0) {
4670                     vm_deviceunplug($vmid, $conf, 'tablet');
4671                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4672                 }
4673             } elsif ($opt =~ m/^usb\d+$/) {
4674                 die "skip\n";
4675                 # since we cannot reliably hot unplug usb devices we disable it for now
4676                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4677                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4678                 #die "skip\n" if !$d;
4679                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4680             } elsif ($opt eq 'vcpus') {
4681                 die "skip\n" if !$hotplug_features->{cpu};
4682                 qemu_cpu_hotplug($vmid, $conf, $value);
4683             } elsif ($opt eq 'balloon') {
4684                 # enable/disable balloning device is not hotpluggable
4685                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4686                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4687                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4688
4689                 # allow manual ballooning if shares is set to zero
4690                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4691                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4692                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4693                 }
4694             } elsif ($opt =~ m/^net(\d+)$/) {
4695                 # some changes can be done without hotplug
4696                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4697                                     $vmid, $opt, $value, $arch, $machine_type);
4698             } elsif (is_valid_drivename($opt)) {
4699                 die "skip\n" if $opt eq 'efidisk0';
4700                 # some changes can be done without hotplug
4701                 my $drive = parse_drive($opt, $value);
4702                 if (drive_is_cloudinit($drive)) {
4703                     &$apply_pending_cloudinit($opt, $value);
4704                 }
4705                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4706                                      $vmid, $opt, $value, $arch, $machine_type);
4707             } elsif ($opt =~ m/^memory$/) { #dimms
4708                 die "skip\n" if !$hotplug_features->{memory};
4709                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4710             } elsif ($opt eq 'cpuunits') {
4711                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4712             } elsif ($opt eq 'cpulimit') {
4713                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4714                 $cgroup->change_cpu_quota($cpulimit, 100000);
4715             } else {
4716                 die "skip\n";  # skip non-hot-pluggable options
4717             }
4718         };
4719         if (my $err = $@) {
4720             &$add_error($opt, $err) if $err ne "skip\n";
4721         } else {
4722             $conf->{$opt} = $value;
4723             delete $conf->{pending}->{$opt};
4724         }
4725     }
4726
4727     PVE::QemuConfig->write_config($vmid, $conf);
4728 }
4729
4730 sub try_deallocate_drive {
4731     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4732
4733     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4734         my $volid = $drive->{file};
4735         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4736             my $sid = PVE::Storage::parse_volume_id($volid);
4737             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4738
4739             # check if the disk is really unused
4740             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4741                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4742             PVE::Storage::vdisk_free($storecfg, $volid);
4743             return 1;
4744         } else {
4745             # If vm is not owner of this disk remove from config
4746             return 1;
4747         }
4748     }
4749
4750     return;
4751 }
4752
4753 sub vmconfig_delete_or_detach_drive {
4754     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4755
4756     my $drive = parse_drive($opt, $conf->{$opt});
4757
4758     my $rpcenv = PVE::RPCEnvironment::get();
4759     my $authuser = $rpcenv->get_user();
4760
4761     if ($force) {
4762         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4763         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4764     } else {
4765         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4766     }
4767 }
4768
4769
4770
4771 sub vmconfig_apply_pending {
4772     my ($vmid, $conf, $storecfg, $errors) = @_;
4773
4774     my $add_apply_error = sub {
4775         my ($opt, $msg) = @_;
4776         my $err_msg = "unable to apply pending change $opt : $msg";
4777         $errors->{$opt} = $err_msg;
4778         warn $err_msg;
4779     };
4780
4781     # cold plug
4782
4783     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4784     foreach my $opt (sort keys %$pending_delete_hash) {
4785         my $force = $pending_delete_hash->{$opt}->{force};
4786         eval {
4787             if ($opt =~ m/^unused/) {
4788                 die "internal error";
4789             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4790                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4791             }
4792         };
4793         if (my $err = $@) {
4794             $add_apply_error->($opt, $err);
4795         } else {
4796             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4797             delete $conf->{$opt};
4798         }
4799     }
4800
4801     PVE::QemuConfig->cleanup_pending($conf);
4802
4803     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4804         next if $opt eq 'delete'; # just to be sure
4805         eval {
4806             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4807                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4808             }
4809         };
4810         if (my $err = $@) {
4811             $add_apply_error->($opt, $err);
4812         } else {
4813             $conf->{$opt} = delete $conf->{pending}->{$opt};
4814         }
4815     }
4816
4817     # write all changes at once to avoid unnecessary i/o
4818     PVE::QemuConfig->write_config($vmid, $conf);
4819 }
4820
4821 sub vmconfig_update_net {
4822     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4823
4824     my $newnet = parse_net($value);
4825
4826     if ($conf->{$opt}) {
4827         my $oldnet = parse_net($conf->{$opt});
4828
4829         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4830             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4831             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4832             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4833
4834             # for non online change, we try to hot-unplug
4835             die "skip\n" if !$hotplug;
4836             vm_deviceunplug($vmid, $conf, $opt);
4837         } else {
4838
4839             die "internal error" if $opt !~ m/net(\d+)/;
4840             my $iface = "tap${vmid}i$1";
4841
4842             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4843                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4844                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4845                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4846                 PVE::Network::tap_unplug($iface);
4847
4848                 if ($have_sdn) {
4849                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4850                 } else {
4851                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4852                 }
4853             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4854                 # Rate can be applied on its own but any change above needs to
4855                 # include the rate in tap_plug since OVS resets everything.
4856                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4857             }
4858
4859             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4860                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4861             }
4862
4863             return 1;
4864         }
4865     }
4866
4867     if ($hotplug) {
4868         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4869     } else {
4870         die "skip\n";
4871     }
4872 }
4873
4874 sub vmconfig_update_disk {
4875     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4876
4877     my $drive = parse_drive($opt, $value);
4878
4879     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4880         my $media = $drive->{media} || 'disk';
4881         my $oldmedia = $old_drive->{media} || 'disk';
4882         die "unable to change media type\n" if $media ne $oldmedia;
4883
4884         if (!drive_is_cdrom($old_drive)) {
4885
4886             if ($drive->{file} ne $old_drive->{file}) {
4887
4888                 die "skip\n" if !$hotplug;
4889
4890                 # unplug and register as unused
4891                 vm_deviceunplug($vmid, $conf, $opt);
4892                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4893
4894             } else {
4895                 # update existing disk
4896
4897                 # skip non hotpluggable value
4898                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4899                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4900                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4901                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4902                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4903                     die "skip\n";
4904                 }
4905
4906                 # apply throttle
4907                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4908                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4909                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4910                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4911                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4912                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4913                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4914                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4915                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4916                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4917                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4918                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4919                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4920                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4921                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4922                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4923                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4924                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4925
4926                     qemu_block_set_io_throttle(
4927                         $vmid,"drive-$opt",
4928                         ($drive->{mbps} || 0)*1024*1024,
4929                         ($drive->{mbps_rd} || 0)*1024*1024,
4930                         ($drive->{mbps_wr} || 0)*1024*1024,
4931                          $drive->{iops} || 0,
4932                          $drive->{iops_rd} || 0,
4933                          $drive->{iops_wr} || 0,
4934                         ($drive->{mbps_max} || 0)*1024*1024,
4935                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4936                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4937                          $drive->{iops_max} || 0,
4938                          $drive->{iops_rd_max} || 0,
4939                          $drive->{iops_wr_max} || 0,
4940                          $drive->{bps_max_length} || 1,
4941                          $drive->{bps_rd_max_length} || 1,
4942                          $drive->{bps_wr_max_length} || 1,
4943                          $drive->{iops_max_length} || 1,
4944                          $drive->{iops_rd_max_length} || 1,
4945                          $drive->{iops_wr_max_length} || 1,
4946                     );
4947
4948                 }
4949
4950                 return 1;
4951             }
4952
4953         } else { # cdrom
4954
4955             if ($drive->{file} eq 'none') {
4956                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4957                 if (drive_is_cloudinit($old_drive)) {
4958                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4959                 }
4960             } else {
4961                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4962
4963                 # force eject if locked
4964                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4965
4966                 if ($path) {
4967                     mon_cmd($vmid, "blockdev-change-medium",
4968                         id => "$opt", filename => "$path");
4969                 }
4970             }
4971
4972             return 1;
4973         }
4974     }
4975
4976     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4977     # hotplug new disks
4978     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4979     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4980 }
4981
4982 # called in locked context by incoming migration
4983 sub vm_migrate_get_nbd_disks {
4984     my ($storecfg, $conf, $replicated_volumes) = @_;
4985
4986     my $local_volumes = {};
4987     PVE::QemuConfig->foreach_volume($conf, sub {
4988         my ($ds, $drive) = @_;
4989
4990         return if drive_is_cdrom($drive);
4991
4992         my $volid = $drive->{file};
4993
4994         return if !$volid;
4995
4996         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4997
4998         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4999         return if $scfg->{shared};
5000
5001         # replicated disks re-use existing state via bitmap
5002         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5003         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5004     });
5005     return $local_volumes;
5006 }
5007
5008 # called in locked context by incoming migration
5009 sub vm_migrate_alloc_nbd_disks {
5010     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5011
5012     my $format = undef;
5013
5014     my $nbd = {};
5015     foreach my $opt (sort keys %$source_volumes) {
5016         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5017
5018         if ($use_existing) {
5019             $nbd->{$opt}->{drivestr} = print_drive($drive);
5020             $nbd->{$opt}->{volid} = $volid;
5021             $nbd->{$opt}->{replicated} = 1;
5022             next;
5023         }
5024
5025         # If a remote storage is specified and the format of the original
5026         # volume is not available there, fall back to the default format.
5027         # Otherwise use the same format as the original.
5028         if (!$storagemap->{identity}) {
5029             $storeid = map_storage($storagemap, $storeid);
5030             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5031             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5032             my $fileFormat = qemu_img_format($scfg, $volname);
5033             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5034         } else {
5035             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5036             $format = qemu_img_format($scfg, $volname);
5037         }
5038
5039         my $size = $drive->{size} / 1024;
5040         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5041         my $newdrive = $drive;
5042         $newdrive->{format} = $format;
5043         $newdrive->{file} = $newvolid;
5044         my $drivestr = print_drive($newdrive);
5045         $nbd->{$opt}->{drivestr} = $drivestr;
5046         $nbd->{$opt}->{volid} = $newvolid;
5047     }
5048
5049     return $nbd;
5050 }
5051
5052 # see vm_start_nolock for parameters, additionally:
5053 # migrate_opts:
5054 #   storagemap = parsed storage map for allocating NBD disks
5055 sub vm_start {
5056     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5057
5058     return PVE::QemuConfig->lock_config($vmid, sub {
5059         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5060
5061         die "you can't start a vm if it's a template\n"
5062             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5063
5064         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5065         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5066
5067         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5068
5069         if ($has_backup_lock && $running) {
5070             # a backup is currently running, attempt to start the guest in the
5071             # existing QEMU instance
5072             return vm_resume($vmid);
5073         }
5074
5075         PVE::QemuConfig->check_lock($conf)
5076             if !($params->{skiplock} || $has_suspended_lock);
5077
5078         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5079
5080         die "VM $vmid already running\n" if $running;
5081
5082         if (my $storagemap = $migrate_opts->{storagemap}) {
5083             my $replicated = $migrate_opts->{replicated_volumes};
5084             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5085             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5086
5087             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5088                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5089             }
5090         }
5091
5092         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5093     });
5094 }
5095
5096
5097 # params:
5098 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5099 #   skiplock => 0/1, skip checking for config lock
5100 #   skiptemplate => 0/1, skip checking whether VM is template
5101 #   forcemachine => to force Qemu machine (rollback/migration)
5102 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5103 #   timeout => in seconds
5104 #   paused => start VM in paused state (backup)
5105 #   resume => resume from hibernation
5106 #   pbs-backing => {
5107 #      sata0 => {
5108 #         repository
5109 #         snapshot
5110 #         keyfile
5111 #         archive
5112 #      },
5113 #      virtio2 => ...
5114 #   }
5115 # migrate_opts:
5116 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5117 #   migratedfrom => source node
5118 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5119 #   network => CIDR of migration network
5120 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5121 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5122 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5123 sub vm_start_nolock {
5124     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5125
5126     my $statefile = $params->{statefile};
5127     my $resume = $params->{resume};
5128
5129     my $migratedfrom = $migrate_opts->{migratedfrom};
5130     my $migration_type = $migrate_opts->{type};
5131
5132     my $res = {};
5133
5134     # clean up leftover reboot request files
5135     eval { clear_reboot_request($vmid); };
5136     warn $@ if $@;
5137
5138     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5139         vmconfig_apply_pending($vmid, $conf, $storecfg);
5140         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5141     }
5142
5143     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5144
5145     my $defaults = load_defaults();
5146
5147     # set environment variable useful inside network script
5148     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5149
5150     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5151
5152     my $forcemachine = $params->{forcemachine};
5153     my $forcecpu = $params->{forcecpu};
5154     if ($resume) {
5155         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5156         $forcemachine = $conf->{runningmachine};
5157         $forcecpu = $conf->{runningcpu};
5158         print "Resuming suspended VM\n";
5159     }
5160
5161     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5162         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5163
5164     my $migration_ip;
5165     my $get_migration_ip = sub {
5166         my ($nodename) = @_;
5167
5168         return $migration_ip if defined($migration_ip);
5169
5170         my $cidr = $migrate_opts->{network};
5171
5172         if (!defined($cidr)) {
5173             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5174             $cidr = $dc_conf->{migration}->{network};
5175         }
5176
5177         if (defined($cidr)) {
5178             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5179
5180             die "could not get IP: no address configured on local " .
5181                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5182
5183             die "could not get IP: multiple addresses configured on local " .
5184                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5185
5186             $migration_ip = @$ips[0];
5187         }
5188
5189         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5190             if !defined($migration_ip);
5191
5192         return $migration_ip;
5193     };
5194
5195     my $migrate_uri;
5196     if ($statefile) {
5197         if ($statefile eq 'tcp') {
5198             my $localip = "localhost";
5199             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5200             my $nodename = nodename();
5201
5202             if (!defined($migration_type)) {
5203                 if (defined($datacenterconf->{migration}->{type})) {
5204                     $migration_type = $datacenterconf->{migration}->{type};
5205                 } else {
5206                     $migration_type = 'secure';
5207                 }
5208             }
5209
5210             if ($migration_type eq 'insecure') {
5211                 $localip = $get_migration_ip->($nodename);
5212                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5213             }
5214
5215             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5216             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5217             $migrate_uri = "tcp:${localip}:${migrate_port}";
5218             push @$cmd, '-incoming', $migrate_uri;
5219             push @$cmd, '-S';
5220
5221         } elsif ($statefile eq 'unix') {
5222             # should be default for secure migrations as a ssh TCP forward
5223             # tunnel is not deterministic reliable ready and fails regurarly
5224             # to set up in time, so use UNIX socket forwards
5225             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5226             unlink $socket_addr;
5227
5228             $migrate_uri = "unix:$socket_addr";
5229
5230             push @$cmd, '-incoming', $migrate_uri;
5231             push @$cmd, '-S';
5232
5233         } elsif (-e $statefile) {
5234             push @$cmd, '-loadstate', $statefile;
5235         } else {
5236             my $statepath = PVE::Storage::path($storecfg, $statefile);
5237             push @$vollist, $statefile;
5238             push @$cmd, '-loadstate', $statepath;
5239         }
5240     } elsif ($params->{paused}) {
5241         push @$cmd, '-S';
5242     }
5243
5244     # host pci devices
5245     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5246       my $d = parse_hostpci($conf->{"hostpci$i"});
5247       next if !$d;
5248       my $pcidevices = $d->{pciid};
5249       foreach my $pcidevice (@$pcidevices) {
5250             my $pciid = $pcidevice->{id};
5251
5252             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5253             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5254             die "no pci device info for device '$pciid'\n" if !$info;
5255
5256             if ($d->{mdev}) {
5257                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5258                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5259             } else {
5260                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5261                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5262                 die "can't reset PCI device '$pciid'\n"
5263                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5264             }
5265       }
5266     }
5267
5268     PVE::Storage::activate_volumes($storecfg, $vollist);
5269
5270     eval {
5271         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5272             outfunc => sub {}, errfunc => sub {});
5273     };
5274     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5275     # timeout should be more than enough here...
5276     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5277
5278     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5279                                               : $defaults->{cpuunits};
5280
5281     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5282     my %run_params = (
5283         timeout => $statefile ? undef : $start_timeout,
5284         umask => 0077,
5285         noerr => 1,
5286     );
5287
5288     # when migrating, prefix QEMU output so other side can pick up any
5289     # errors that might occur and show the user
5290     if ($migratedfrom) {
5291         $run_params{quiet} = 1;
5292         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5293     }
5294
5295     my %properties = (
5296         Slice => 'qemu.slice',
5297         KillMode => 'process',
5298         SendSIGKILL => 0,
5299         TimeoutStopUSec => ULONG_MAX, # infinity
5300     );
5301
5302     if (PVE::CGroup::cgroup_mode() == 2) {
5303         $properties{CPUWeight} = $cpuunits;
5304     } else {
5305         $properties{CPUShares} = $cpuunits;
5306     }
5307
5308     if (my $cpulimit = $conf->{cpulimit}) {
5309         $properties{CPUQuota} = int($cpulimit * 100);
5310     }
5311     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5312
5313     my $run_qemu = sub {
5314         PVE::Tools::run_fork sub {
5315             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5316
5317             my $exitcode = run_command($cmd, %run_params);
5318             die "QEMU exited with code $exitcode\n" if $exitcode;
5319         };
5320     };
5321
5322     if ($conf->{hugepages}) {
5323
5324         my $code = sub {
5325             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5326             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5327
5328             PVE::QemuServer::Memory::hugepages_mount();
5329             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5330
5331             eval { $run_qemu->() };
5332             if (my $err = $@) {
5333                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5334                     if !$conf->{keephugepages};
5335                 die $err;
5336             }
5337
5338             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5339                 if !$conf->{keephugepages};
5340         };
5341         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5342
5343     } else {
5344         eval { $run_qemu->() };
5345     }
5346
5347     if (my $err = $@) {
5348         # deactivate volumes if start fails
5349         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5350         die "start failed: $err";
5351     }
5352
5353     print "migration listens on $migrate_uri\n" if $migrate_uri;
5354     $res->{migrate_uri} = $migrate_uri;
5355
5356     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5357         eval { mon_cmd($vmid, "cont"); };
5358         warn $@ if $@;
5359     }
5360
5361     #start nbd server for storage migration
5362     if (my $nbd = $migrate_opts->{nbd}) {
5363         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5364
5365         my $migrate_storage_uri;
5366         # nbd_protocol_version > 0 for unix socket support
5367         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5368             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5369             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5370             $migrate_storage_uri = "nbd:unix:$socket_path";
5371         } else {
5372             my $nodename = nodename();
5373             my $localip = $get_migration_ip->($nodename);
5374             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5375             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5376
5377             mon_cmd($vmid, "nbd-server-start", addr => {
5378                 type => 'inet',
5379                 data => {
5380                     host => "${localip}",
5381                     port => "${storage_migrate_port}",
5382                 },
5383             });
5384             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5385             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5386         }
5387
5388         $res->{migrate_storage_uri} = $migrate_storage_uri;
5389
5390         foreach my $opt (sort keys %$nbd) {
5391             my $drivestr = $nbd->{$opt}->{drivestr};
5392             my $volid = $nbd->{$opt}->{volid};
5393             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5394             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5395             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5396             print "re-using replicated volume: $opt - $volid\n"
5397                 if $nbd->{$opt}->{replicated};
5398
5399             $res->{drives}->{$opt} = $nbd->{$opt};
5400             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5401         }
5402     }
5403
5404     if ($migratedfrom) {
5405         eval {
5406             set_migration_caps($vmid);
5407         };
5408         warn $@ if $@;
5409
5410         if ($spice_port) {
5411             print "spice listens on port $spice_port\n";
5412             $res->{spice_port} = $spice_port;
5413             if ($migrate_opts->{spice_ticket}) {
5414                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5415                     $migrate_opts->{spice_ticket});
5416                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5417             }
5418         }
5419
5420     } else {
5421         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5422             if !$statefile && $conf->{balloon};
5423
5424         foreach my $opt (keys %$conf) {
5425             next if $opt !~  m/^net\d+$/;
5426             my $nicconf = parse_net($conf->{$opt});
5427             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5428         }
5429     }
5430
5431     mon_cmd($vmid, 'qom-set',
5432                 path => "machine/peripheral/balloon0",
5433                 property => "guest-stats-polling-interval",
5434                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5435
5436     if ($resume) {
5437         print "Resumed VM, removing state\n";
5438         if (my $vmstate = $conf->{vmstate}) {
5439             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5440             PVE::Storage::vdisk_free($storecfg, $vmstate);
5441         }
5442         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5443         PVE::QemuConfig->write_config($vmid, $conf);
5444     }
5445
5446     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5447
5448     return $res;
5449 }
5450
5451 sub vm_commandline {
5452     my ($storecfg, $vmid, $snapname) = @_;
5453
5454     my $conf = PVE::QemuConfig->load_config($vmid);
5455     my $forcemachine;
5456     my $forcecpu;
5457
5458     if ($snapname) {
5459         my $snapshot = $conf->{snapshots}->{$snapname};
5460         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5461
5462         # check for machine or CPU overrides in snapshot
5463         $forcemachine = $snapshot->{runningmachine};
5464         $forcecpu = $snapshot->{runningcpu};
5465
5466         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5467
5468         $conf = $snapshot;
5469     }
5470
5471     my $defaults = load_defaults();
5472
5473     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5474         $forcemachine, $forcecpu);
5475
5476     return PVE::Tools::cmd2string($cmd);
5477 }
5478
5479 sub vm_reset {
5480     my ($vmid, $skiplock) = @_;
5481
5482     PVE::QemuConfig->lock_config($vmid, sub {
5483
5484         my $conf = PVE::QemuConfig->load_config($vmid);
5485
5486         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5487
5488         mon_cmd($vmid, "system_reset");
5489     });
5490 }
5491
5492 sub get_vm_volumes {
5493     my ($conf) = @_;
5494
5495     my $vollist = [];
5496     foreach_volid($conf, sub {
5497         my ($volid, $attr) = @_;
5498
5499         return if $volid =~ m|^/|;
5500
5501         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5502         return if !$sid;
5503
5504         push @$vollist, $volid;
5505     });
5506
5507     return $vollist;
5508 }
5509
5510 sub vm_stop_cleanup {
5511     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5512
5513     eval {
5514
5515         if (!$keepActive) {
5516             my $vollist = get_vm_volumes($conf);
5517             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5518         }
5519
5520         foreach my $ext (qw(mon qmp pid vnc qga)) {
5521             unlink "/var/run/qemu-server/${vmid}.$ext";
5522         }
5523
5524         if ($conf->{ivshmem}) {
5525             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5526             # just delete it for now, VMs which have this already open do not
5527             # are affected, but new VMs will get a separated one. If this
5528             # becomes an issue we either add some sort of ref-counting or just
5529             # add a "don't delete on stop" flag to the ivshmem format.
5530             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5531         }
5532
5533         foreach my $key (keys %$conf) {
5534             next if $key !~ m/^hostpci(\d+)$/;
5535             my $hostpciindex = $1;
5536             my $d = parse_hostpci($conf->{$key});
5537             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5538
5539             foreach my $pci (@{$d->{pciid}}) {
5540                 my $pciid = $pci->{id};
5541                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5542             }
5543         }
5544
5545         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5546     };
5547     warn $@ if $@; # avoid errors - just warn
5548 }
5549
5550 # call only in locked context
5551 sub _do_vm_stop {
5552     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5553
5554     my $pid = check_running($vmid, $nocheck);
5555     return if !$pid;
5556
5557     my $conf;
5558     if (!$nocheck) {
5559         $conf = PVE::QemuConfig->load_config($vmid);
5560         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5561         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5562             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5563             $timeout = $opts->{down} if $opts->{down};
5564         }
5565         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5566     }
5567
5568     eval {
5569         if ($shutdown) {
5570             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5571                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5572             } else {
5573                 mon_cmd($vmid, "system_powerdown");
5574             }
5575         } else {
5576             mon_cmd($vmid, "quit");
5577         }
5578     };
5579     my $err = $@;
5580
5581     if (!$err) {
5582         $timeout = 60 if !defined($timeout);
5583
5584         my $count = 0;
5585         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5586             $count++;
5587             sleep 1;
5588         }
5589
5590         if ($count >= $timeout) {
5591             if ($force) {
5592                 warn "VM still running - terminating now with SIGTERM\n";
5593                 kill 15, $pid;
5594             } else {
5595                 die "VM quit/powerdown failed - got timeout\n";
5596             }
5597         } else {
5598             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5599             return;
5600         }
5601     } else {
5602        if (!check_running($vmid, $nocheck)) {
5603             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5604             return;
5605        }
5606        if ($force) {
5607             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5608             kill 15, $pid;
5609         } else {
5610             die "VM quit/powerdown failed\n";
5611         }
5612     }
5613
5614     # wait again
5615     $timeout = 10;
5616
5617     my $count = 0;
5618     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5619         $count++;
5620         sleep 1;
5621     }
5622
5623     if ($count >= $timeout) {
5624         warn "VM still running - terminating now with SIGKILL\n";
5625         kill 9, $pid;
5626         sleep 1;
5627     }
5628
5629     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5630 }
5631
5632 # Note: use $nocheck to skip tests if VM configuration file exists.
5633 # We need that when migration VMs to other nodes (files already moved)
5634 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5635 sub vm_stop {
5636     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5637
5638     $force = 1 if !defined($force) && !$shutdown;
5639
5640     if ($migratedfrom){
5641         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5642         kill 15, $pid if $pid;
5643         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5644         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5645         return;
5646     }
5647
5648     PVE::QemuConfig->lock_config($vmid, sub {
5649         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5650    });
5651 }
5652
5653 sub vm_reboot {
5654     my ($vmid, $timeout) = @_;
5655
5656     PVE::QemuConfig->lock_config($vmid, sub {
5657         eval {
5658
5659             # only reboot if running, as qmeventd starts it again on a stop event
5660             return if !check_running($vmid);
5661
5662             create_reboot_request($vmid);
5663
5664             my $storecfg = PVE::Storage::config();
5665             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5666
5667         };
5668         if (my $err = $@) {
5669             # avoid that the next normal shutdown will be confused for a reboot
5670             clear_reboot_request($vmid);
5671             die $err;
5672         }
5673    });
5674 }
5675
5676 # note: if using the statestorage parameter, the caller has to check privileges
5677 sub vm_suspend {
5678     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5679
5680     my $conf;
5681     my $path;
5682     my $storecfg;
5683     my $vmstate;
5684
5685     PVE::QemuConfig->lock_config($vmid, sub {
5686
5687         $conf = PVE::QemuConfig->load_config($vmid);
5688
5689         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5690         PVE::QemuConfig->check_lock($conf)
5691             if !($skiplock || $is_backing_up);
5692
5693         die "cannot suspend to disk during backup\n"
5694             if $is_backing_up && $includestate;
5695
5696         if ($includestate) {
5697             $conf->{lock} = 'suspending';
5698             my $date = strftime("%Y-%m-%d", localtime(time()));
5699             $storecfg = PVE::Storage::config();
5700             if (!$statestorage) {
5701                 $statestorage = find_vmstate_storage($conf, $storecfg);
5702                 # check permissions for the storage
5703                 my $rpcenv = PVE::RPCEnvironment::get();
5704                 if ($rpcenv->{type} ne 'cli') {
5705                     my $authuser = $rpcenv->get_user();
5706                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5707                 }
5708             }
5709
5710
5711             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5712                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5713             $path = PVE::Storage::path($storecfg, $vmstate);
5714             PVE::QemuConfig->write_config($vmid, $conf);
5715         } else {
5716             mon_cmd($vmid, "stop");
5717         }
5718     });
5719
5720     if ($includestate) {
5721         # save vm state
5722         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5723
5724         eval {
5725             set_migration_caps($vmid, 1);
5726             mon_cmd($vmid, "savevm-start", statefile => $path);
5727             for(;;) {
5728                 my $state = mon_cmd($vmid, "query-savevm");
5729                 if (!$state->{status}) {
5730                     die "savevm not active\n";
5731                 } elsif ($state->{status} eq 'active') {
5732                     sleep(1);
5733                     next;
5734                 } elsif ($state->{status} eq 'completed') {
5735                     print "State saved, quitting\n";
5736                     last;
5737                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5738                     die "query-savevm failed with error '$state->{error}'\n"
5739                 } else {
5740                     die "query-savevm returned status '$state->{status}'\n";
5741                 }
5742             }
5743         };
5744         my $err = $@;
5745
5746         PVE::QemuConfig->lock_config($vmid, sub {
5747             $conf = PVE::QemuConfig->load_config($vmid);
5748             if ($err) {
5749                 # cleanup, but leave suspending lock, to indicate something went wrong
5750                 eval {
5751                     mon_cmd($vmid, "savevm-end");
5752                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5753                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5754                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5755                     PVE::QemuConfig->write_config($vmid, $conf);
5756                 };
5757                 warn $@ if $@;
5758                 die $err;
5759             }
5760
5761             die "lock changed unexpectedly\n"
5762                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5763
5764             mon_cmd($vmid, "quit");
5765             $conf->{lock} = 'suspended';
5766             PVE::QemuConfig->write_config($vmid, $conf);
5767         });
5768     }
5769 }
5770
5771 sub vm_resume {
5772     my ($vmid, $skiplock, $nocheck) = @_;
5773
5774     PVE::QemuConfig->lock_config($vmid, sub {
5775         my $res = mon_cmd($vmid, 'query-status');
5776         my $resume_cmd = 'cont';
5777         my $reset = 0;
5778
5779         if ($res->{status}) {
5780             return if $res->{status} eq 'running'; # job done, go home
5781             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5782             $reset = 1 if $res->{status} eq 'shutdown';
5783         }
5784
5785         if (!$nocheck) {
5786
5787             my $conf = PVE::QemuConfig->load_config($vmid);
5788
5789             PVE::QemuConfig->check_lock($conf)
5790                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5791         }
5792
5793         if ($reset) {
5794             # required if a VM shuts down during a backup and we get a resume
5795             # request before the backup finishes for example
5796             mon_cmd($vmid, "system_reset");
5797         }
5798         mon_cmd($vmid, $resume_cmd);
5799     });
5800 }
5801
5802 sub vm_sendkey {
5803     my ($vmid, $skiplock, $key) = @_;
5804
5805     PVE::QemuConfig->lock_config($vmid, sub {
5806
5807         my $conf = PVE::QemuConfig->load_config($vmid);
5808
5809         # there is no qmp command, so we use the human monitor command
5810         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5811         die $res if $res ne '';
5812     });
5813 }
5814
5815 # vzdump restore implementaion
5816
5817 sub tar_archive_read_firstfile {
5818     my $archive = shift;
5819
5820     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5821
5822     # try to detect archive type first
5823     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5824         die "unable to open file '$archive'\n";
5825     my $firstfile = <$fh>;
5826     kill 15, $pid;
5827     close $fh;
5828
5829     die "ERROR: archive contaions no data\n" if !$firstfile;
5830     chomp $firstfile;
5831
5832     return $firstfile;
5833 }
5834
5835 sub tar_restore_cleanup {
5836     my ($storecfg, $statfile) = @_;
5837
5838     print STDERR "starting cleanup\n";
5839
5840     if (my $fd = IO::File->new($statfile, "r")) {
5841         while (defined(my $line = <$fd>)) {
5842             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5843                 my $volid = $2;
5844                 eval {
5845                     if ($volid =~ m|^/|) {
5846                         unlink $volid || die 'unlink failed\n';
5847                     } else {
5848                         PVE::Storage::vdisk_free($storecfg, $volid);
5849                     }
5850                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5851                 };
5852                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5853             } else {
5854                 print STDERR "unable to parse line in statfile - $line";
5855             }
5856         }
5857         $fd->close();
5858     }
5859 }
5860
5861 sub restore_file_archive {
5862     my ($archive, $vmid, $user, $opts) = @_;
5863
5864     return restore_vma_archive($archive, $vmid, $user, $opts)
5865         if $archive eq '-';
5866
5867     my $info = PVE::Storage::archive_info($archive);
5868     my $format = $opts->{format} // $info->{format};
5869     my $comp = $info->{compression};
5870
5871     # try to detect archive format
5872     if ($format eq 'tar') {
5873         return restore_tar_archive($archive, $vmid, $user, $opts);
5874     } else {
5875         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5876     }
5877 }
5878
5879 # hepler to remove disks that will not be used after restore
5880 my $restore_cleanup_oldconf = sub {
5881     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5882
5883     PVE::QemuConfig->foreach_volume($oldconf, sub {
5884         my ($ds, $drive) = @_;
5885
5886         return if drive_is_cdrom($drive, 1);
5887
5888         my $volid = $drive->{file};
5889         return if !$volid || $volid =~ m|^/|;
5890
5891         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5892         return if !$path || !$owner || ($owner != $vmid);
5893
5894         # Note: only delete disk we want to restore
5895         # other volumes will become unused
5896         if ($virtdev_hash->{$ds}) {
5897             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5898             if (my $err = $@) {
5899                 warn $err;
5900             }
5901         }
5902     });
5903
5904     # delete vmstate files, after the restore we have no snapshots anymore
5905     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5906         my $snap = $oldconf->{snapshots}->{$snapname};
5907         if ($snap->{vmstate}) {
5908             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5909             if (my $err = $@) {
5910                 warn $err;
5911             }
5912         }
5913     }
5914 };
5915
5916 # Helper to parse vzdump backup device hints
5917 #
5918 # $rpcenv: Environment, used to ckeck storage permissions
5919 # $user: User ID, to check storage permissions
5920 # $storecfg: Storage configuration
5921 # $fh: the file handle for reading the configuration
5922 # $devinfo: should contain device sizes for all backu-up'ed devices
5923 # $options: backup options (pool, default storage)
5924 #
5925 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5926 my $parse_backup_hints = sub {
5927     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5928
5929     my $virtdev_hash = {};
5930
5931     while (defined(my $line = <$fh>)) {
5932         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5933             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5934             die "archive does not contain data for drive '$virtdev'\n"
5935                 if !$devinfo->{$devname};
5936
5937             if (defined($options->{storage})) {
5938                 $storeid = $options->{storage} || 'local';
5939             } elsif (!$storeid) {
5940                 $storeid = 'local';
5941             }
5942             $format = 'raw' if !$format;
5943             $devinfo->{$devname}->{devname} = $devname;
5944             $devinfo->{$devname}->{virtdev} = $virtdev;
5945             $devinfo->{$devname}->{format} = $format;
5946             $devinfo->{$devname}->{storeid} = $storeid;
5947
5948             # check permission on storage
5949             my $pool = $options->{pool}; # todo: do we need that?
5950             if ($user ne 'root@pam') {
5951                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5952             }
5953
5954             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5955         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5956             my $virtdev = $1;
5957             my $drive = parse_drive($virtdev, $2);
5958             if (drive_is_cloudinit($drive)) {
5959                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5960                 $storeid = $options->{storage} if defined ($options->{storage});
5961                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5962                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5963
5964                 $virtdev_hash->{$virtdev} = {
5965                     format => $format,
5966                     storeid => $storeid,
5967                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5968                     is_cloudinit => 1,
5969                 };
5970             }
5971         }
5972     }
5973
5974     return $virtdev_hash;
5975 };
5976
5977 # Helper to allocate and activate all volumes required for a restore
5978 #
5979 # $storecfg: Storage configuration
5980 # $virtdev_hash: as returned by parse_backup_hints()
5981 #
5982 # Returns: { $virtdev => $volid }
5983 my $restore_allocate_devices = sub {
5984     my ($storecfg, $virtdev_hash, $vmid) = @_;
5985
5986     my $map = {};
5987     foreach my $virtdev (sort keys %$virtdev_hash) {
5988         my $d = $virtdev_hash->{$virtdev};
5989         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5990         my $storeid = $d->{storeid};
5991         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5992
5993         # test if requested format is supported
5994         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5995         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5996         $d->{format} = $defFormat if !$supported;
5997
5998         my $name;
5999         if ($d->{is_cloudinit}) {
6000             $name = "vm-$vmid-cloudinit";
6001             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6002             if ($scfg->{path}) {
6003                 $name .= ".$d->{format}";
6004             }
6005         }
6006
6007         my $volid = PVE::Storage::vdisk_alloc(
6008             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6009
6010         print STDERR "new volume ID is '$volid'\n";
6011         $d->{volid} = $volid;
6012
6013         PVE::Storage::activate_volumes($storecfg, [$volid]);
6014
6015         $map->{$virtdev} = $volid;
6016     }
6017
6018     return $map;
6019 };
6020
6021 sub restore_update_config_line {
6022     my ($cookie, $map, $line, $unique) = @_;
6023
6024     return '' if $line =~ m/^\#qmdump\#/;
6025     return '' if $line =~ m/^\#vzdump\#/;
6026     return '' if $line =~ m/^lock:/;
6027     return '' if $line =~ m/^unused\d+:/;
6028     return '' if $line =~ m/^parent:/;
6029
6030     my $res = '';
6031
6032     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6033     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6034         # try to convert old 1.X settings
6035         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6036         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6037             my ($model, $macaddr) = split(/\=/, $devconfig);
6038             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6039             my $net = {
6040                 model => $model,
6041                 bridge => "vmbr$ind",
6042                 macaddr => $macaddr,
6043             };
6044             my $netstr = print_net($net);
6045
6046             $res .= "net$cookie->{netcount}: $netstr\n";
6047             $cookie->{netcount}++;
6048         }
6049     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6050         my ($id, $netstr) = ($1, $2);
6051         my $net = parse_net($netstr);
6052         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6053         $netstr = print_net($net);
6054         $res .= "$id: $netstr\n";
6055     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6056         my $virtdev = $1;
6057         my $value = $3;
6058         my $di = parse_drive($virtdev, $value);
6059         if (defined($di->{backup}) && !$di->{backup}) {
6060             $res .= "#$line";
6061         } elsif ($map->{$virtdev}) {
6062             delete $di->{format}; # format can change on restore
6063             $di->{file} = $map->{$virtdev};
6064             $value = print_drive($di);
6065             $res .= "$virtdev: $value\n";
6066         } else {
6067             $res .= $line;
6068         }
6069     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6070         my $vmgenid = $1;
6071         if ($vmgenid ne '0') {
6072             # always generate a new vmgenid if there was a valid one setup
6073             $vmgenid = generate_uuid();
6074         }
6075         $res .= "vmgenid: $vmgenid\n";
6076     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6077         my ($uuid, $uuid_str);
6078         UUID::generate($uuid);
6079         UUID::unparse($uuid, $uuid_str);
6080         my $smbios1 = parse_smbios1($2);
6081         $smbios1->{uuid} = $uuid_str;
6082         $res .= $1.print_smbios1($smbios1)."\n";
6083     } else {
6084         $res .= $line;
6085     }
6086
6087     return $res;
6088 }
6089
6090 my $restore_deactivate_volumes = sub {
6091     my ($storecfg, $devinfo) = @_;
6092
6093     my $vollist = [];
6094     foreach my $devname (keys %$devinfo) {
6095         my $volid = $devinfo->{$devname}->{volid};
6096         push @$vollist, $volid if $volid;
6097     }
6098
6099     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6100 };
6101
6102 my $restore_destroy_volumes = sub {
6103     my ($storecfg, $devinfo) = @_;
6104
6105     foreach my $devname (keys %$devinfo) {
6106         my $volid = $devinfo->{$devname}->{volid};
6107         next if !$volid;
6108         eval {
6109             if ($volid =~ m|^/|) {
6110                 unlink $volid || die 'unlink failed\n';
6111             } else {
6112                 PVE::Storage::vdisk_free($storecfg, $volid);
6113             }
6114             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6115         };
6116         print STDERR "unable to cleanup '$volid' - $@" if $@;
6117     }
6118 };
6119
6120 sub scan_volids {
6121     my ($cfg, $vmid) = @_;
6122
6123     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6124
6125     my $volid_hash = {};
6126     foreach my $storeid (keys %$info) {
6127         foreach my $item (@{$info->{$storeid}}) {
6128             next if !($item->{volid} && $item->{size});
6129             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6130             $volid_hash->{$item->{volid}} = $item;
6131         }
6132     }
6133
6134     return $volid_hash;
6135 }
6136
6137 sub update_disk_config {
6138     my ($vmid, $conf, $volid_hash) = @_;
6139
6140     my $changes;
6141     my $prefix = "VM $vmid";
6142
6143     # used and unused disks
6144     my $referenced = {};
6145
6146     # Note: it is allowed to define multiple storages with same path (alias), so
6147     # we need to check both 'volid' and real 'path' (two different volid can point
6148     # to the same path).
6149
6150     my $referencedpath = {};
6151
6152     # update size info
6153     PVE::QemuConfig->foreach_volume($conf, sub {
6154         my ($opt, $drive) = @_;
6155
6156         my $volid = $drive->{file};
6157         return if !$volid;
6158         my $volume = $volid_hash->{$volid};
6159
6160         # mark volid as "in-use" for next step
6161         $referenced->{$volid} = 1;
6162         if ($volume && (my $path = $volume->{path})) {
6163             $referencedpath->{$path} = 1;
6164         }
6165
6166         return if drive_is_cdrom($drive);
6167         return if !$volume;
6168
6169         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6170         if (defined($updated)) {
6171             $changes = 1;
6172             $conf->{$opt} = print_drive($updated);
6173             print "$prefix ($opt): $msg\n";
6174         }
6175     });
6176
6177     # remove 'unusedX' entry if volume is used
6178     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6179         my ($opt, $drive) = @_;
6180
6181         my $volid = $drive->{file};
6182         return if !$volid;
6183
6184         my $path;
6185         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6186         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6187             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6188             $changes = 1;
6189             delete $conf->{$opt};
6190         }
6191
6192         $referenced->{$volid} = 1;
6193         $referencedpath->{$path} = 1 if $path;
6194     });
6195
6196     foreach my $volid (sort keys %$volid_hash) {
6197         next if $volid =~ m/vm-$vmid-state-/;
6198         next if $referenced->{$volid};
6199         my $path = $volid_hash->{$volid}->{path};
6200         next if !$path; # just to be sure
6201         next if $referencedpath->{$path};
6202         $changes = 1;
6203         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6204         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6205         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6206     }
6207
6208     return $changes;
6209 }
6210
6211 sub rescan {
6212     my ($vmid, $nolock, $dryrun) = @_;
6213
6214     my $cfg = PVE::Storage::config();
6215
6216     print "rescan volumes...\n";
6217     my $volid_hash = scan_volids($cfg, $vmid);
6218
6219     my $updatefn =  sub {
6220         my ($vmid) = @_;
6221
6222         my $conf = PVE::QemuConfig->load_config($vmid);
6223
6224         PVE::QemuConfig->check_lock($conf);
6225
6226         my $vm_volids = {};
6227         foreach my $volid (keys %$volid_hash) {
6228             my $info = $volid_hash->{$volid};
6229             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6230         }
6231
6232         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6233
6234         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6235     };
6236
6237     if (defined($vmid)) {
6238         if ($nolock) {
6239             &$updatefn($vmid);
6240         } else {
6241             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6242         }
6243     } else {
6244         my $vmlist = config_list();
6245         foreach my $vmid (keys %$vmlist) {
6246             if ($nolock) {
6247                 &$updatefn($vmid);
6248             } else {
6249                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6250             }
6251         }
6252     }
6253 }
6254
6255 sub restore_proxmox_backup_archive {
6256     my ($archive, $vmid, $user, $options) = @_;
6257
6258     my $storecfg = PVE::Storage::config();
6259
6260     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6261     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6262
6263     my $fingerprint = $scfg->{fingerprint};
6264     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6265
6266     my $repo = PVE::PBSClient::get_repository($scfg);
6267
6268     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6269     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6270     local $ENV{PBS_PASSWORD} = $password;
6271     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6272
6273     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6274         PVE::Storage::parse_volname($storecfg, $archive);
6275
6276     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6277
6278     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6279
6280     my $tmpdir = "/var/tmp/vzdumptmp$$";
6281     rmtree $tmpdir;
6282     mkpath $tmpdir;
6283
6284     my $conffile = PVE::QemuConfig->config_file($vmid);
6285      # disable interrupts (always do cleanups)
6286     local $SIG{INT} =
6287         local $SIG{TERM} =
6288         local $SIG{QUIT} =
6289         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6290
6291     # Note: $oldconf is undef if VM does not exists
6292     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6293     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6294     my $new_conf_raw = '';
6295
6296     my $rpcenv = PVE::RPCEnvironment::get();
6297     my $devinfo = {};
6298
6299     eval {
6300         # enable interrupts
6301         local $SIG{INT} =
6302             local $SIG{TERM} =
6303             local $SIG{QUIT} =
6304             local $SIG{HUP} =
6305             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6306
6307         my $cfgfn = "$tmpdir/qemu-server.conf";
6308         my $firewall_config_fn = "$tmpdir/fw.conf";
6309         my $index_fn = "$tmpdir/index.json";
6310
6311         my $cmd = "restore";
6312
6313         my $param = [$pbs_backup_name, "index.json", $index_fn];
6314         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6315         my $index = PVE::Tools::file_get_contents($index_fn);
6316         $index = decode_json($index);
6317
6318         # print Dumper($index);
6319         foreach my $info (@{$index->{files}}) {
6320             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6321                 my $devname = $1;
6322                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6323                     $devinfo->{$devname}->{size} = $1;
6324                 } else {
6325                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6326                 }
6327             }
6328         }
6329
6330         my $is_qemu_server_backup = scalar(
6331             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6332         );
6333         if (!$is_qemu_server_backup) {
6334             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6335         }
6336         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6337
6338         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6339         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6340
6341         if ($has_firewall_config) {
6342             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6343             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6344
6345             my $pve_firewall_dir = '/etc/pve/firewall';
6346             mkdir $pve_firewall_dir; # make sure the dir exists
6347             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6348         }
6349
6350         my $fh = IO::File->new($cfgfn, "r") ||
6351             die "unable to read qemu-server.conf - $!\n";
6352
6353         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6354
6355         # fixme: rate limit?
6356
6357         # create empty/temp config
6358         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6359
6360         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6361
6362         # allocate volumes
6363         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6364
6365         if (!$options->{live}) {
6366             foreach my $virtdev (sort keys %$virtdev_hash) {
6367                 my $d = $virtdev_hash->{$virtdev};
6368                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6369
6370                 my $volid = $d->{volid};
6371
6372                 my $path = PVE::Storage::path($storecfg, $volid);
6373
6374                 my $pbs_restore_cmd = [
6375                     '/usr/bin/pbs-restore',
6376                     '--repository', $repo,
6377                     $pbs_backup_name,
6378                     "$d->{devname}.img.fidx",
6379                     $path,
6380                     '--verbose',
6381                     ];
6382
6383                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6384                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6385
6386                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6387                     push @$pbs_restore_cmd, '--skip-zero';
6388                 }
6389
6390                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6391                 print "restore proxmox backup image: $dbg_cmdstring\n";
6392                 run_command($pbs_restore_cmd);
6393             }
6394         }
6395
6396         $fh->seek(0, 0) || die "seek failed - $!\n";
6397
6398         my $cookie = { netcount => 0 };
6399         while (defined(my $line = <$fh>)) {
6400             $new_conf_raw .= restore_update_config_line(
6401                 $cookie,
6402                 $map,
6403                 $line,
6404                 $options->{unique},
6405             );
6406         }
6407
6408         $fh->close();
6409     };
6410     my $err = $@;
6411
6412     if ($err || !$options->{live}) {
6413         $restore_deactivate_volumes->($storecfg, $devinfo);
6414     }
6415
6416     rmtree $tmpdir;
6417
6418     if ($err) {
6419         $restore_destroy_volumes->($storecfg, $devinfo);
6420         die $err;
6421     }
6422
6423     if ($options->{live}) {
6424         # keep lock during live-restore
6425         $new_conf_raw .= "\nlock: create";
6426     }
6427
6428     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6429
6430     PVE::Cluster::cfs_update(); # make sure we read new file
6431
6432     eval { rescan($vmid, 1); };
6433     warn $@ if $@;
6434
6435     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6436
6437     if ($options->{live}) {
6438         # enable interrupts
6439         local $SIG{INT} =
6440             local $SIG{TERM} =
6441             local $SIG{QUIT} =
6442             local $SIG{HUP} =
6443             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6444
6445         my $conf = PVE::QemuConfig->load_config($vmid);
6446         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6447
6448         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6449
6450         PVE::QemuConfig->remove_lock($vmid, "create");
6451     }
6452 }
6453
6454 sub pbs_live_restore {
6455     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6456
6457     print "starting VM for live-restore\n";
6458     print "repository: '$repo', snapshot: '$snap'\n";
6459
6460     my $pbs_backing = {};
6461     for my $ds (keys %$restored_disks) {
6462         $ds =~ m/^drive-(.*)$/;
6463         my $confname = $1;
6464         $pbs_backing->{$confname} = {
6465             repository => $repo,
6466             snapshot => $snap,
6467             archive => "$ds.img.fidx",
6468         };
6469         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6470
6471         my $drive = parse_drive($confname, $conf->{$confname});
6472         print "restoring '$ds' to '$drive->{file}'\n";
6473     }
6474
6475     my $drives_streamed = 0;
6476     eval {
6477         # make sure HA doesn't interrupt our restore by stopping the VM
6478         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6479             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6480         }
6481
6482         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6483         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6484         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6485
6486         my $qmeventd_fd = register_qmeventd_handle($vmid);
6487
6488         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6489         # this will effectively collapse the backing image chain consisting of
6490         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6491         # removes itself once all backing images vanish with 'auto-remove=on')
6492         my $jobs = {};
6493         for my $ds (sort keys %$restored_disks) {
6494             my $job_id = "restore-$ds";
6495             mon_cmd($vmid, 'block-stream',
6496                 'job-id' => $job_id,
6497                 device => "$ds",
6498             );
6499             $jobs->{$job_id} = {};
6500         }
6501
6502         mon_cmd($vmid, 'cont');
6503         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6504
6505         print "restore-drive jobs finished successfully, removing all tracking block devices"
6506             ." to disconnect from Proxmox Backup Server\n";
6507
6508         for my $ds (sort keys %$restored_disks) {
6509             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6510         }
6511
6512         close($qmeventd_fd);
6513     };
6514
6515     my $err = $@;
6516
6517     if ($err) {
6518         warn "An error occured during live-restore: $err\n";
6519         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6520         die "live-restore failed\n";
6521     }
6522 }
6523
6524 sub restore_vma_archive {
6525     my ($archive, $vmid, $user, $opts, $comp) = @_;
6526
6527     my $readfrom = $archive;
6528
6529     my $cfg = PVE::Storage::config();
6530     my $commands = [];
6531     my $bwlimit = $opts->{bwlimit};
6532
6533     my $dbg_cmdstring = '';
6534     my $add_pipe = sub {
6535         my ($cmd) = @_;
6536         push @$commands, $cmd;
6537         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6538         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6539         $readfrom = '-';
6540     };
6541
6542     my $input = undef;
6543     if ($archive eq '-') {
6544         $input = '<&STDIN';
6545     } else {
6546         # If we use a backup from a PVE defined storage we also consider that
6547         # storage's rate limit:
6548         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6549         if (defined($volid)) {
6550             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6551             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6552             if ($readlimit) {
6553                 print STDERR "applying read rate limit: $readlimit\n";
6554                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6555                 $add_pipe->($cstream);
6556             }
6557         }
6558     }
6559
6560     if ($comp) {
6561         my $info = PVE::Storage::decompressor_info('vma', $comp);
6562         my $cmd = $info->{decompressor};
6563         push @$cmd, $readfrom;
6564         $add_pipe->($cmd);
6565     }
6566
6567     my $tmpdir = "/var/tmp/vzdumptmp$$";
6568     rmtree $tmpdir;
6569
6570     # disable interrupts (always do cleanups)
6571     local $SIG{INT} =
6572         local $SIG{TERM} =
6573         local $SIG{QUIT} =
6574         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6575
6576     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6577     POSIX::mkfifo($mapfifo, 0600);
6578     my $fifofh;
6579     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6580
6581     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6582
6583     my $oldtimeout;
6584     my $timeout = 5;
6585
6586     my $devinfo = {};
6587
6588     my $rpcenv = PVE::RPCEnvironment::get();
6589
6590     my $conffile = PVE::QemuConfig->config_file($vmid);
6591
6592     # Note: $oldconf is undef if VM does not exist
6593     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6594     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6595     my $new_conf_raw = '';
6596
6597     my %storage_limits;
6598
6599     my $print_devmap = sub {
6600         my $cfgfn = "$tmpdir/qemu-server.conf";
6601
6602         # we can read the config - that is already extracted
6603         my $fh = IO::File->new($cfgfn, "r") ||
6604             die "unable to read qemu-server.conf - $!\n";
6605
6606         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6607         if (-f $fwcfgfn) {
6608             my $pve_firewall_dir = '/etc/pve/firewall';
6609             mkdir $pve_firewall_dir; # make sure the dir exists
6610             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6611         }
6612
6613         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6614
6615         foreach my $info (values %{$virtdev_hash}) {
6616             my $storeid = $info->{storeid};
6617             next if defined($storage_limits{$storeid});
6618
6619             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6620             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6621             $storage_limits{$storeid} = $limit * 1024;
6622         }
6623
6624         foreach my $devname (keys %$devinfo) {
6625             die "found no device mapping information for device '$devname'\n"
6626                 if !$devinfo->{$devname}->{virtdev};
6627         }
6628
6629         # create empty/temp config
6630         if ($oldconf) {
6631             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6632             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6633         }
6634
6635         # allocate volumes
6636         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6637
6638         # print restore information to $fifofh
6639         foreach my $virtdev (sort keys %$virtdev_hash) {
6640             my $d = $virtdev_hash->{$virtdev};
6641             next if $d->{is_cloudinit}; # no need to restore cloudinit
6642
6643             my $storeid = $d->{storeid};
6644             my $volid = $d->{volid};
6645
6646             my $map_opts = '';
6647             if (my $limit = $storage_limits{$storeid}) {
6648                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6649             }
6650
6651             my $write_zeros = 1;
6652             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6653                 $write_zeros = 0;
6654             }
6655
6656             my $path = PVE::Storage::path($cfg, $volid);
6657
6658             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6659
6660             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6661         }
6662
6663         $fh->seek(0, 0) || die "seek failed - $!\n";
6664
6665         my $cookie = { netcount => 0 };
6666         while (defined(my $line = <$fh>)) {
6667             $new_conf_raw .= restore_update_config_line(
6668                 $cookie,
6669                 $map,
6670                 $line,
6671                 $opts->{unique},
6672             );
6673         }
6674
6675         $fh->close();
6676     };
6677
6678     eval {
6679         # enable interrupts
6680         local $SIG{INT} =
6681             local $SIG{TERM} =
6682             local $SIG{QUIT} =
6683             local $SIG{HUP} =
6684             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6685         local $SIG{ALRM} = sub { die "got timeout\n"; };
6686
6687         $oldtimeout = alarm($timeout);
6688
6689         my $parser = sub {
6690             my $line = shift;
6691
6692             print "$line\n";
6693
6694             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6695                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6696                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6697             } elsif ($line =~ m/^CTIME: /) {
6698                 # we correctly received the vma config, so we can disable
6699                 # the timeout now for disk allocation (set to 10 minutes, so
6700                 # that we always timeout if something goes wrong)
6701                 alarm(600);
6702                 &$print_devmap();
6703                 print $fifofh "done\n";
6704                 my $tmp = $oldtimeout || 0;
6705                 $oldtimeout = undef;
6706                 alarm($tmp);
6707                 close($fifofh);
6708                 $fifofh = undef;
6709             }
6710         };
6711
6712         print "restore vma archive: $dbg_cmdstring\n";
6713         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6714     };
6715     my $err = $@;
6716
6717     alarm($oldtimeout) if $oldtimeout;
6718
6719     $restore_deactivate_volumes->($cfg, $devinfo);
6720
6721     close($fifofh) if $fifofh;
6722     unlink $mapfifo;
6723     rmtree $tmpdir;
6724
6725     if ($err) {
6726         $restore_destroy_volumes->($cfg, $devinfo);
6727         die $err;
6728     }
6729
6730     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6731
6732     PVE::Cluster::cfs_update(); # make sure we read new file
6733
6734     eval { rescan($vmid, 1); };
6735     warn $@ if $@;
6736
6737     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6738 }
6739
6740 sub restore_tar_archive {
6741     my ($archive, $vmid, $user, $opts) = @_;
6742
6743     if ($archive ne '-') {
6744         my $firstfile = tar_archive_read_firstfile($archive);
6745         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6746             if $firstfile ne 'qemu-server.conf';
6747     }
6748
6749     my $storecfg = PVE::Storage::config();
6750
6751     # avoid zombie disks when restoring over an existing VM -> cleanup first
6752     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6753     # skiplock=1 because qmrestore has set the 'create' lock itself already
6754     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6755     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6756
6757     my $tocmd = "/usr/lib/qemu-server/qmextract";
6758
6759     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6760     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6761     $tocmd .= ' --prealloc' if $opts->{prealloc};
6762     $tocmd .= ' --info' if $opts->{info};
6763
6764     # tar option "xf" does not autodetect compression when read from STDIN,
6765     # so we pipe to zcat
6766     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6767         PVE::Tools::shellquote("--to-command=$tocmd");
6768
6769     my $tmpdir = "/var/tmp/vzdumptmp$$";
6770     mkpath $tmpdir;
6771
6772     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6773     local $ENV{VZDUMP_VMID} = $vmid;
6774     local $ENV{VZDUMP_USER} = $user;
6775
6776     my $conffile = PVE::QemuConfig->config_file($vmid);
6777     my $new_conf_raw = '';
6778
6779     # disable interrupts (always do cleanups)
6780     local $SIG{INT} =
6781         local $SIG{TERM} =
6782         local $SIG{QUIT} =
6783         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6784
6785     eval {
6786         # enable interrupts
6787         local $SIG{INT} =
6788             local $SIG{TERM} =
6789             local $SIG{QUIT} =
6790             local $SIG{HUP} =
6791             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6792
6793         if ($archive eq '-') {
6794             print "extracting archive from STDIN\n";
6795             run_command($cmd, input => "<&STDIN");
6796         } else {
6797             print "extracting archive '$archive'\n";
6798             run_command($cmd);
6799         }
6800
6801         return if $opts->{info};
6802
6803         # read new mapping
6804         my $map = {};
6805         my $statfile = "$tmpdir/qmrestore.stat";
6806         if (my $fd = IO::File->new($statfile, "r")) {
6807             while (defined (my $line = <$fd>)) {
6808                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6809                     $map->{$1} = $2 if $1;
6810                 } else {
6811                     print STDERR "unable to parse line in statfile - $line\n";
6812                 }
6813             }
6814             $fd->close();
6815         }
6816
6817         my $confsrc = "$tmpdir/qemu-server.conf";
6818
6819         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6820
6821         my $cookie = { netcount => 0 };
6822         while (defined (my $line = <$srcfd>)) {
6823             $new_conf_raw .= restore_update_config_line(
6824                 $cookie,
6825                 $map,
6826                 $line,
6827                 $opts->{unique},
6828             );
6829         }
6830
6831         $srcfd->close();
6832     };
6833     if (my $err = $@) {
6834         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6835         die $err;
6836     }
6837
6838     rmtree $tmpdir;
6839
6840     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6841
6842     PVE::Cluster::cfs_update(); # make sure we read new file
6843
6844     eval { rescan($vmid, 1); };
6845     warn $@ if $@;
6846 };
6847
6848 sub foreach_storage_used_by_vm {
6849     my ($conf, $func) = @_;
6850
6851     my $sidhash = {};
6852
6853     PVE::QemuConfig->foreach_volume($conf, sub {
6854         my ($ds, $drive) = @_;
6855         return if drive_is_cdrom($drive);
6856
6857         my $volid = $drive->{file};
6858
6859         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6860         $sidhash->{$sid} = $sid if $sid;
6861     });
6862
6863     foreach my $sid (sort keys %$sidhash) {
6864         &$func($sid);
6865     }
6866 }
6867
6868 my $qemu_snap_storage = {
6869     rbd => 1,
6870 };
6871 sub do_snapshots_with_qemu {
6872     my ($storecfg, $volid) = @_;
6873
6874     my $storage_name = PVE::Storage::parse_volume_id($volid);
6875     my $scfg = $storecfg->{ids}->{$storage_name};
6876     die "could not find storage '$storage_name'\n" if !defined($scfg);
6877
6878     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6879         return 1;
6880     }
6881
6882     if ($volid =~ m/\.(qcow2|qed)$/){
6883         return 1;
6884     }
6885
6886     return;
6887 }
6888
6889 sub qga_check_running {
6890     my ($vmid, $nowarn) = @_;
6891
6892     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6893     if ($@) {
6894         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6895         return 0;
6896     }
6897     return 1;
6898 }
6899
6900 sub template_create {
6901     my ($vmid, $conf, $disk) = @_;
6902
6903     my $storecfg = PVE::Storage::config();
6904
6905     PVE::QemuConfig->foreach_volume($conf, sub {
6906         my ($ds, $drive) = @_;
6907
6908         return if drive_is_cdrom($drive);
6909         return if $disk && $ds ne $disk;
6910
6911         my $volid = $drive->{file};
6912         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6913
6914         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6915         $drive->{file} = $voliddst;
6916         $conf->{$ds} = print_drive($drive);
6917         PVE::QemuConfig->write_config($vmid, $conf);
6918     });
6919 }
6920
6921 sub convert_iscsi_path {
6922     my ($path) = @_;
6923
6924     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6925         my $portal = $1;
6926         my $target = $2;
6927         my $lun = $3;
6928
6929         my $initiator_name = get_initiator_name();
6930
6931         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6932                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6933     }
6934
6935     die "cannot convert iscsi path '$path', unkown format\n";
6936 }
6937
6938 sub qemu_img_convert {
6939     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6940
6941     my $storecfg = PVE::Storage::config();
6942     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6943     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6944
6945     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6946
6947     my $cachemode;
6948     my $src_path;
6949     my $src_is_iscsi = 0;
6950     my $src_format;
6951
6952     if ($src_storeid) {
6953         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6954         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6955         $src_format = qemu_img_format($src_scfg, $src_volname);
6956         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6957         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6958         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6959     } elsif (-f $src_volid) {
6960         $src_path = $src_volid;
6961         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6962             $src_format = $1;
6963         }
6964     }
6965
6966     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6967
6968     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6969     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6970     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6971     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6972
6973     my $cmd = [];
6974     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6975     push @$cmd, '-l', "snapshot.name=$snapname"
6976         if $snapname && $src_format && $src_format eq "qcow2";
6977     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6978     push @$cmd, '-T', $cachemode if defined($cachemode);
6979
6980     if ($src_is_iscsi) {
6981         push @$cmd, '--image-opts';
6982         $src_path = convert_iscsi_path($src_path);
6983     } elsif ($src_format) {
6984         push @$cmd, '-f', $src_format;
6985     }
6986
6987     if ($dst_is_iscsi) {
6988         push @$cmd, '--target-image-opts';
6989         $dst_path = convert_iscsi_path($dst_path);
6990     } else {
6991         push @$cmd, '-O', $dst_format;
6992     }
6993
6994     push @$cmd, $src_path;
6995
6996     if (!$dst_is_iscsi && $is_zero_initialized) {
6997         push @$cmd, "zeroinit:$dst_path";
6998     } else {
6999         push @$cmd, $dst_path;
7000     }
7001
7002     my $parser = sub {
7003         my $line = shift;
7004         if($line =~ m/\((\S+)\/100\%\)/){
7005             my $percent = $1;
7006             my $transferred = int($size * $percent / 100);
7007             my $total_h = render_bytes($size, 1);
7008             my $transferred_h = render_bytes($transferred, 1);
7009
7010             print "transferred $transferred_h of $total_h ($percent%)\n";
7011         }
7012
7013     };
7014
7015     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7016     my $err = $@;
7017     die "copy failed: $err" if $err;
7018 }
7019
7020 sub qemu_img_format {
7021     my ($scfg, $volname) = @_;
7022
7023     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7024         return $1;
7025     } else {
7026         return "raw";
7027     }
7028 }
7029
7030 sub qemu_drive_mirror {
7031     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7032
7033     $jobs = {} if !$jobs;
7034
7035     my $qemu_target;
7036     my $format;
7037     $jobs->{"drive-$drive"} = {};
7038
7039     if ($dst_volid =~ /^nbd:/) {
7040         $qemu_target = $dst_volid;
7041         $format = "nbd";
7042     } else {
7043         my $storecfg = PVE::Storage::config();
7044         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7045
7046         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7047
7048         $format = qemu_img_format($dst_scfg, $dst_volname);
7049
7050         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7051
7052         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7053     }
7054
7055     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7056     $opts->{format} = $format if $format;
7057
7058     if (defined($src_bitmap)) {
7059         $opts->{sync} = 'incremental';
7060         $opts->{bitmap} = $src_bitmap;
7061         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7062     }
7063
7064     if (defined($bwlimit)) {
7065         $opts->{speed} = $bwlimit * 1024;
7066         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7067     } else {
7068         print "drive mirror is starting for drive-$drive\n";
7069     }
7070
7071     # if a job already runs for this device we get an error, catch it for cleanup
7072     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7073     if (my $err = $@) {
7074         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7075         warn "$@\n" if $@;
7076         die "mirroring error: $err\n";
7077     }
7078
7079     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7080 }
7081
7082 # $completion can be either
7083 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7084 # 'cancel': wait until all jobs are ready, block-job-cancel them
7085 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7086 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7087 sub qemu_drive_mirror_monitor {
7088     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7089
7090     $completion //= 'complete';
7091     $op //= "mirror";
7092
7093     eval {
7094         my $err_complete = 0;
7095
7096         my $starttime = time ();
7097         while (1) {
7098             die "block job ('$op') timed out\n" if $err_complete > 300;
7099
7100             my $stats = mon_cmd($vmid, "query-block-jobs");
7101             my $ctime = time();
7102
7103             my $running_jobs = {};
7104             for my $stat (@$stats) {
7105                 next if $stat->{type} ne $op;
7106                 $running_jobs->{$stat->{device}} = $stat;
7107             }
7108
7109             my $readycounter = 0;
7110
7111             for my $job_id (sort keys %$jobs) {
7112                 my $job = $running_jobs->{$job_id};
7113
7114                 my $vanished = !defined($job);
7115                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7116                 if($complete || ($vanished && $completion eq 'auto')) {
7117                     print "$job_id: $op-job finished\n";
7118                     delete $jobs->{$job_id};
7119                     next;
7120                 }
7121
7122                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7123
7124                 my $busy = $job->{busy};
7125                 my $ready = $job->{ready};
7126                 if (my $total = $job->{len}) {
7127                     my $transferred = $job->{offset} || 0;
7128                     my $remaining = $total - $transferred;
7129                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7130
7131                     my $duration = $ctime - $starttime;
7132                     my $total_h = render_bytes($total, 1);
7133                     my $transferred_h = render_bytes($transferred, 1);
7134
7135                     my $status = sprintf(
7136                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7137                         render_duration($duration),
7138                     );
7139
7140                     if ($ready) {
7141                         if ($busy) {
7142                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7143                         } else {
7144                             $status .= ", ready";
7145                         }
7146                     }
7147                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7148                     $jobs->{$job_id}->{ready} = $ready;
7149                 }
7150
7151                 $readycounter++ if $job->{ready};
7152             }
7153
7154             last if scalar(keys %$jobs) == 0;
7155
7156             if ($readycounter == scalar(keys %$jobs)) {
7157                 print "all '$op' jobs are ready\n";
7158
7159                 # do the complete later (or has already been done)
7160                 last if $completion eq 'skip' || $completion eq 'auto';
7161
7162                 if ($vmiddst && $vmiddst != $vmid) {
7163                     my $agent_running = $qga && qga_check_running($vmid);
7164                     if ($agent_running) {
7165                         print "freeze filesystem\n";
7166                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7167                     } else {
7168                         print "suspend vm\n";
7169                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7170                     }
7171
7172                     # if we clone a disk for a new target vm, we don't switch the disk
7173                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7174
7175                     if ($agent_running) {
7176                         print "unfreeze filesystem\n";
7177                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7178                     } else {
7179                         print "resume vm\n";
7180                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7181                     }
7182
7183                     last;
7184                 } else {
7185
7186                     for my $job_id (sort keys %$jobs) {
7187                         # try to switch the disk if source and destination are on the same guest
7188                         print "$job_id: Completing block job_id...\n";
7189
7190                         my $op;
7191                         if ($completion eq 'complete') {
7192                             $op = 'block-job-complete';
7193                         } elsif ($completion eq 'cancel') {
7194                             $op = 'block-job-cancel';
7195                         } else {
7196                             die "invalid completion value: $completion\n";
7197                         }
7198                         eval { mon_cmd($vmid, $op, device => $job_id) };
7199                         if ($@ =~ m/cannot be completed/) {
7200                             print "$job_id: block job cannot be completed, trying again.\n";
7201                             $err_complete++;
7202                         }else {
7203                             print "$job_id: Completed successfully.\n";
7204                             $jobs->{$job_id}->{complete} = 1;
7205                         }
7206                     }
7207                 }
7208             }
7209             sleep 1;
7210         }
7211     };
7212     my $err = $@;
7213
7214     if ($err) {
7215         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7216         die "block job ($op) error: $err";
7217     }
7218 }
7219
7220 sub qemu_blockjobs_cancel {
7221     my ($vmid, $jobs) = @_;
7222
7223     foreach my $job (keys %$jobs) {
7224         print "$job: Cancelling block job\n";
7225         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7226         $jobs->{$job}->{cancel} = 1;
7227     }
7228
7229     while (1) {
7230         my $stats = mon_cmd($vmid, "query-block-jobs");
7231
7232         my $running_jobs = {};
7233         foreach my $stat (@$stats) {
7234             $running_jobs->{$stat->{device}} = $stat;
7235         }
7236
7237         foreach my $job (keys %$jobs) {
7238
7239             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7240                 print "$job: Done.\n";
7241                 delete $jobs->{$job};
7242             }
7243         }
7244
7245         last if scalar(keys %$jobs) == 0;
7246
7247         sleep 1;
7248     }
7249 }
7250
7251 sub clone_disk {
7252     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7253         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7254
7255     my $newvolid;
7256
7257     if (!$full) {
7258         print "create linked clone of drive $drivename ($drive->{file})\n";
7259         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7260         push @$newvollist, $newvolid;
7261     } else {
7262
7263         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7264         $storeid = $storage if $storage;
7265
7266         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7267
7268         print "create full clone of drive $drivename ($drive->{file})\n";
7269         my $name = undef;
7270         my $size = undef;
7271         if (drive_is_cloudinit($drive)) {
7272             $name = "vm-$newvmid-cloudinit";
7273             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7274             if ($scfg->{path}) {
7275                 $name .= ".$dst_format";
7276             }
7277             $snapname = undef;
7278             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7279         } elsif ($drivename eq 'efidisk0') {
7280             $size = get_efivars_size($conf);
7281         } else {
7282             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7283         }
7284         $newvolid = PVE::Storage::vdisk_alloc(
7285             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7286         );
7287         push @$newvollist, $newvolid;
7288
7289         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7290
7291         if (drive_is_cloudinit($drive)) {
7292             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7293             # if this is the case, we have to complete any block-jobs still there from
7294             # previous drive-mirrors
7295             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7296                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7297             }
7298             goto no_data_clone;
7299         }
7300
7301         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7302         if (!$running || $snapname) {
7303             # TODO: handle bwlimits
7304             if ($drivename eq 'efidisk0') {
7305                 # the relevant data on the efidisk may be smaller than the source
7306                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7307                 # that is given by the OVMF_VARS.fd
7308                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7309                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7310
7311                 # better for Ceph if block size is not too small, see bug #3324
7312                 my $bs = 1024*1024;
7313
7314                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7315                     "if=$src_path", "of=$dst_path"]);
7316             } else {
7317                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7318             }
7319         } else {
7320
7321             my $kvmver = get_running_qemu_version ($vmid);
7322             if (!min_version($kvmver, 2, 7)) {
7323                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7324                     if $drive->{iothread};
7325             }
7326
7327             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7328                 $completion, $qga, $bwlimit);
7329         }
7330     }
7331
7332 no_data_clone:
7333     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7334
7335     my $disk = $drive;
7336     $disk->{format} = undef;
7337     $disk->{file} = $newvolid;
7338     $disk->{size} = $size if defined($size);
7339
7340     return $disk;
7341 }
7342
7343 sub get_running_qemu_version {
7344     my ($vmid) = @_;
7345     my $res = mon_cmd($vmid, "query-version");
7346     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7347 }
7348
7349 sub qemu_use_old_bios_files {
7350     my ($machine_type) = @_;
7351
7352     return if !$machine_type;
7353
7354     my $use_old_bios_files = undef;
7355
7356     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7357         $machine_type = $1;
7358         $use_old_bios_files = 1;
7359     } else {
7360         my $version = extract_version($machine_type, kvm_user_version());
7361         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7362         # load new efi bios files on migration. So this hack is required to allow
7363         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7364         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7365         $use_old_bios_files = !min_version($version, 2, 4);
7366     }
7367
7368     return ($use_old_bios_files, $machine_type);
7369 }
7370
7371 sub get_efivars_size {
7372     my ($conf) = @_;
7373     my $arch = get_vm_arch($conf);
7374     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7375     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7376     return -s $ovmf_vars;
7377 }
7378
7379 sub update_efidisk_size {
7380     my ($conf) = @_;
7381
7382     return if !defined($conf->{efidisk0});
7383
7384     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7385     $disk->{size} = get_efivars_size($conf);
7386     $conf->{efidisk0} = print_drive($disk);
7387
7388     return;
7389 }
7390
7391 sub create_efidisk($$$$$) {
7392     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7393
7394     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7395     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7396
7397     my $vars_size_b = -s $ovmf_vars;
7398     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7399     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7400     PVE::Storage::activate_volumes($storecfg, [$volid]);
7401
7402     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7403     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7404
7405     return ($volid, $size/1024);
7406 }
7407
7408 sub vm_iothreads_list {
7409     my ($vmid) = @_;
7410
7411     my $res = mon_cmd($vmid, 'query-iothreads');
7412
7413     my $iothreads = {};
7414     foreach my $iothread (@$res) {
7415         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7416     }
7417
7418     return $iothreads;
7419 }
7420
7421 sub scsihw_infos {
7422     my ($conf, $drive) = @_;
7423
7424     my $maxdev = 0;
7425
7426     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7427         $maxdev = 7;
7428     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7429         $maxdev = 1;
7430     } else {
7431         $maxdev = 256;
7432     }
7433
7434     my $controller = int($drive->{index} / $maxdev);
7435     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7436         ? "virtioscsi"
7437         : "scsihw";
7438
7439     return ($maxdev, $controller, $controller_prefix);
7440 }
7441
7442 sub windows_version {
7443     my ($ostype) = @_;
7444
7445     return 0 if !$ostype;
7446
7447     my $winversion = 0;
7448
7449     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7450         $winversion = 5;
7451     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7452         $winversion = 6;
7453     } elsif ($ostype =~ m/^win(\d+)$/) {
7454         $winversion = $1;
7455     }
7456
7457     return $winversion;
7458 }
7459
7460 sub resolve_dst_disk_format {
7461         my ($storecfg, $storeid, $src_volname, $format) = @_;
7462         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7463
7464         if (!$format) {
7465             # if no target format is specified, use the source disk format as hint
7466             if ($src_volname) {
7467                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7468                 $format = qemu_img_format($scfg, $src_volname);
7469             } else {
7470                 return $defFormat;
7471             }
7472         }
7473
7474         # test if requested format is supported - else use default
7475         my $supported = grep { $_ eq $format } @$validFormats;
7476         $format = $defFormat if !$supported;
7477         return $format;
7478 }
7479
7480 # NOTE: if this logic changes, please update docs & possibly gui logic
7481 sub find_vmstate_storage {
7482     my ($conf, $storecfg) = @_;
7483
7484     # first, return storage from conf if set
7485     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7486
7487     my ($target, $shared, $local);
7488
7489     foreach_storage_used_by_vm($conf, sub {
7490         my ($sid) = @_;
7491         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7492         my $dst = $scfg->{shared} ? \$shared : \$local;
7493         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7494     });
7495
7496     # second, use shared storage where VM has at least one disk
7497     # third, use local storage where VM has at least one disk
7498     # fall back to local storage
7499     $target = $shared // $local // 'local';
7500
7501     return $target;
7502 }
7503
7504 sub generate_uuid {
7505     my ($uuid, $uuid_str);
7506     UUID::generate($uuid);
7507     UUID::unparse($uuid, $uuid_str);
7508     return $uuid_str;
7509 }
7510
7511 sub generate_smbios1_uuid {
7512     return "uuid=".generate_uuid();
7513 }
7514
7515 sub nbd_stop {
7516     my ($vmid) = @_;
7517
7518     mon_cmd($vmid, 'nbd-server-stop');
7519 }
7520
7521 sub create_reboot_request {
7522     my ($vmid) = @_;
7523     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7524         or die "failed to create reboot trigger file: $!\n";
7525     close($fh);
7526 }
7527
7528 sub clear_reboot_request {
7529     my ($vmid) = @_;
7530     my $path = "/run/qemu-server/$vmid.reboot";
7531     my $res = 0;
7532
7533     $res = unlink($path);
7534     die "could not remove reboot request for $vmid: $!"
7535         if !$res && $! != POSIX::ENOENT;
7536
7537     return $res;
7538 }
7539
7540 sub bootorder_from_legacy {
7541     my ($conf, $bootcfg) = @_;
7542
7543     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7544     my $bootindex_hash = {};
7545     my $i = 1;
7546     foreach my $o (split(//, $boot)) {
7547         $bootindex_hash->{$o} = $i*100;
7548         $i++;
7549     }
7550
7551     my $bootorder = {};
7552
7553     PVE::QemuConfig->foreach_volume($conf, sub {
7554         my ($ds, $drive) = @_;
7555
7556         if (drive_is_cdrom ($drive, 1)) {
7557             if ($bootindex_hash->{d}) {
7558                 $bootorder->{$ds} = $bootindex_hash->{d};
7559                 $bootindex_hash->{d} += 1;
7560             }
7561         } elsif ($bootindex_hash->{c}) {
7562             $bootorder->{$ds} = $bootindex_hash->{c}
7563                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7564             $bootindex_hash->{c} += 1;
7565         }
7566     });
7567
7568     if ($bootindex_hash->{n}) {
7569         for (my $i = 0; $i < $MAX_NETS; $i++) {
7570             my $netname = "net$i";
7571             next if !$conf->{$netname};
7572             $bootorder->{$netname} = $bootindex_hash->{n};
7573             $bootindex_hash->{n} += 1;
7574         }
7575     }
7576
7577     return $bootorder;
7578 }
7579
7580 # Generate default device list for 'boot: order=' property. Matches legacy
7581 # default boot order, but with explicit device names. This is important, since
7582 # the fallback for when neither 'order' nor the old format is specified relies
7583 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7584 sub get_default_bootdevices {
7585     my ($conf) = @_;
7586
7587     my @ret = ();
7588
7589     # harddisk
7590     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7591     push @ret, $first if $first;
7592
7593     # cdrom
7594     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7595     push @ret, $first if $first;
7596
7597     # network
7598     for (my $i = 0; $i < $MAX_NETS; $i++) {
7599         my $netname = "net$i";
7600         next if !$conf->{$netname};
7601         push @ret, $netname;
7602         last;
7603     }
7604
7605     return \@ret;
7606 }
7607
7608 sub device_bootorder {
7609     my ($conf) = @_;
7610
7611     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7612
7613     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7614
7615     my $bootorder = {};
7616     if (!defined($boot) || $boot->{legacy}) {
7617         $bootorder = bootorder_from_legacy($conf, $boot);
7618     } elsif ($boot->{order}) {
7619         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7620         for my $dev (PVE::Tools::split_list($boot->{order})) {
7621             $bootorder->{$dev} = $i++;
7622         }
7623     }
7624
7625     return $bootorder;
7626 }
7627
7628 sub register_qmeventd_handle {
7629     my ($vmid) = @_;
7630
7631     my $fh;
7632     my $peer = "/var/run/qmeventd.sock";
7633     my $count = 0;
7634
7635     for (;;) {
7636         $count++;
7637         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7638         last if $fh;
7639         if ($! != EINTR && $! != EAGAIN) {
7640             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7641         }
7642         if ($count > 4) {
7643             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7644               . "after $count retries\n";
7645         }
7646         usleep(25000);
7647     }
7648
7649     # send handshake to mark VM as backing up
7650     print $fh to_json({vzdump => {vmid => "$vmid"}});
7651
7652     # return handle to be closed later when inhibit is no longer required
7653     return $fh;
7654 }
7655
7656 # bash completion helper
7657
7658 sub complete_backup_archives {
7659     my ($cmdname, $pname, $cvalue) = @_;
7660
7661     my $cfg = PVE::Storage::config();
7662
7663     my $storeid;
7664
7665     if ($cvalue =~ m/^([^:]+):/) {
7666         $storeid = $1;
7667     }
7668
7669     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7670
7671     my $res = [];
7672     foreach my $id (keys %$data) {
7673         foreach my $item (@{$data->{$id}}) {
7674             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7675             push @$res, $item->{volid} if defined($item->{volid});
7676         }
7677     }
7678
7679     return $res;
7680 }
7681
7682 my $complete_vmid_full = sub {
7683     my ($running) = @_;
7684
7685     my $idlist = vmstatus();
7686
7687     my $res = [];
7688
7689     foreach my $id (keys %$idlist) {
7690         my $d = $idlist->{$id};
7691         if (defined($running)) {
7692             next if $d->{template};
7693             next if $running && $d->{status} ne 'running';
7694             next if !$running && $d->{status} eq 'running';
7695         }
7696         push @$res, $id;
7697
7698     }
7699     return $res;
7700 };
7701
7702 sub complete_vmid {
7703     return &$complete_vmid_full();
7704 }
7705
7706 sub complete_vmid_stopped {
7707     return &$complete_vmid_full(0);
7708 }
7709
7710 sub complete_vmid_running {
7711     return &$complete_vmid_full(1);
7712 }
7713
7714 sub complete_storage {
7715
7716     my $cfg = PVE::Storage::config();
7717     my $ids = $cfg->{ids};
7718
7719     my $res = [];
7720     foreach my $sid (keys %$ids) {
7721         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7722         next if !$ids->{$sid}->{content}->{images};
7723         push @$res, $sid;
7724     }
7725
7726     return $res;
7727 }
7728
7729 sub complete_migration_storage {
7730     my ($cmd, $param, $current_value, $all_args) = @_;
7731
7732     my $targetnode = @$all_args[1];
7733
7734     my $cfg = PVE::Storage::config();
7735     my $ids = $cfg->{ids};
7736
7737     my $res = [];
7738     foreach my $sid (keys %$ids) {
7739         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7740         next if !$ids->{$sid}->{content}->{images};
7741         push @$res, $sid;
7742     }
7743
7744     return $res;
7745 }
7746
7747 sub vm_is_paused {
7748     my ($vmid) = @_;
7749     my $qmpstatus = eval {
7750         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7751         mon_cmd($vmid, "query-status");
7752     };
7753     warn "$@\n" if $@;
7754     return $qmpstatus && $qmpstatus->{status} eq "paused";
7755 }
7756
7757 sub check_volume_storage_type {
7758     my ($storecfg, $vol) = @_;
7759
7760     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7761     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7762     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7763
7764     die "storage '$storeid' does not support content-type '$vtype'\n"
7765         if !$scfg->{content}->{$vtype};
7766
7767     return 1;
7768 }
7769
7770 1;