PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Shown in the web-interface VM's summary."
 369             ." This is saved as comment inside the configuration file.",
 370         maxLength => 1024 * 8,
 371     },
 372     ostype => {
 373         optional => 1,
 374         type => 'string',
 375         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 376         description => "Specify guest operating system.",
 377         verbose_description => <<EODESC,
 378 Specify guest operating system. This is used to enable special
 379 optimization/features for specific operating systems:
 380
 381 [horizontal]
 382 other;; unspecified OS
 383 wxp;; Microsoft Windows XP
 384 w2k;; Microsoft Windows 2000
 385 w2k3;; Microsoft Windows 2003
 386 w2k8;; Microsoft Windows 2008
 387 wvista;; Microsoft Windows Vista
 388 win7;; Microsoft Windows 7
 389 win8;; Microsoft Windows 8/2012/2012r2
 390 win10;; Microsoft Windows 10/2016/2019
 391 l24;; Linux 2.4 Kernel
 392 l26;; Linux 2.6 - 5.X Kernel
 393 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 394 EODESC
 395     },
 396     boot => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-boot',
 399         description => "Specify guest boot order. Use with 'order=', usage with"
 400                      . " no key or 'legacy=' is deprecated.",
 401     },
 402     bootdisk => {
 403         optional => 1,
 404         type => 'string', format => 'pve-qm-bootdisk',
 405         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 406         pattern => '(ide|sata|scsi|virtio)\d+',
 407     },
 408     smp => {
 409         optional => 1,
 410         type => 'integer',
 411         description => "The number of CPUs. Please use option -sockets instead.",
 412         minimum => 1,
 413         default => 1,
 414     },
 415     sockets => {
 416         optional => 1,
 417         type => 'integer',
 418         description => "The number of CPU sockets.",
 419         minimum => 1,
 420         default => 1,
 421     },
 422     cores => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "The number of cores per socket.",
 426         minimum => 1,
 427         default => 1,
 428     },
 429     numa => {
 430         optional => 1,
 431         type => 'boolean',
 432         description => "Enable/disable NUMA.",
 433         default => 0,
 434     },
 435     hugepages => {
 436         optional => 1,
 437         type => 'string',
 438         description => "Enable/disable hugepages memory.",
 439         enum => [qw(any 2 1024)],
 440     },
 441     keephugepages => {
 442         optional => 1,
 443         type => 'boolean',
 444         default => 0,
 445         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 446             ." after VM shutdown and can be used for subsequent starts.",
 447     },
 448     vcpus => {
 449         optional => 1,
 450         type => 'integer',
 451         description => "Number of hotplugged vcpus.",
 452         minimum => 1,
 453         default => 0,
 454     },
 455     acpi => {
 456         optional => 1,
 457         type => 'boolean',
 458         description => "Enable/disable ACPI.",
 459         default => 1,
 460     },
 461     agent => {
 462         optional => 1,
 463         description => "Enable/disable Qemu GuestAgent and its properties.",
 464         type => 'string',
 465         format => $agent_fmt,
 466     },
 467     kvm => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable KVM hardware virtualization.",
 471         default => 1,
 472     },
 473     tdf => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable time drift fix.",
 477         default => 0,
 478     },
 479     localtime => {
 480         optional => 1,
 481         type => 'boolean',
 482         description => "Set the real time clock to local time. This is enabled by default if ostype"
 483             ." indicates a Microsoft OS.",
 484     },
 485     freeze => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 489     },
 490     vga => {
 491         optional => 1,
 492         type => 'string', format => $vga_fmt,
 493         description => "Configure the VGA hardware.",
 494         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 495             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 496             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 497             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 498             ." display server. For win* OS you can select how many independent displays you want,"
 499             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 500             ." using a serial device as terminal.",
 501     },
 502     watchdog => {
 503         optional => 1,
 504         type => 'string', format => 'pve-qm-watchdog',
 505         description => "Create a virtual hardware watchdog device.",
 506         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 507             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 508             ." else the watchdog will reset the guest (or execute the respective action specified)",
 509     },
 510     startdate => {
 511         optional => 1,
 512         type => 'string',
 513         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 514         description => "Set the initial date of the real time clock. Valid format for date are:"
 515             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 516         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 517         default => 'now',
 518     },
 519     startup =>  get_standard_option('pve-startup-order'),
 520     template => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Enable/disable Template.",
 524         default => 0,
 525     },
 526     args => {
 527         optional => 1,
 528         type => 'string',
 529         description => "Arbitrary arguments passed to kvm.",
 530         verbose_description => <<EODESCR,
 531 Arbitrary arguments passed to kvm, for example:
 532
 533 args: -no-reboot -no-hpet
 534
 535 NOTE: this option is for experts only.
 536 EODESCR
 537     },
 538     tablet => {
 539         optional => 1,
 540         type => 'boolean',
 541         default => 1,
 542         description => "Enable/disable the USB tablet device.",
 543         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 544             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 545             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 546             ." may consider disabling this to save some context switches. This is turned off by"
 547             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 548     },
 549     migrate_speed => {
 550         optional => 1,
 551         type => 'integer',
 552         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 553         minimum => 0,
 554         default => 0,
 555     },
 556     migrate_downtime => {
 557         optional => 1,
 558         type => 'number',
 559         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 560         minimum => 0,
 561         default => 0.1,
 562     },
 563     cdrom => {
 564         optional => 1,
 565         type => 'string', format => 'pve-qm-ide',
 566         typetext => '<volume>',
 567         description => "This is an alias for option -ide2",
 568     },
 569     cpu => {
 570         optional => 1,
 571         description => "Emulated CPU type.",
 572         type => 'string',
 573         format => 'pve-vm-cpu-conf',
 574     },
 575     parent => get_standard_option('pve-snapshot-name', {
 576         optional => 1,
 577         description => "Parent snapshot name. This is used internally, and should not be modified.",
 578     }),
 579     snaptime => {
 580         optional => 1,
 581         description => "Timestamp for snapshots.",
 582         type => 'integer',
 583         minimum => 0,
 584     },
 585     vmstate => {
 586         optional => 1,
 587         type => 'string', format => 'pve-volume-id',
 588         description => "Reference to a volume which stores the VM state. This is used internally"
 589             ." for snapshots.",
 590     },
 591     vmstatestorage => get_standard_option('pve-storage-id', {
 592         description => "Default storage for VM state volumes/files.",
 593         optional => 1,
 594     }),
 595     runningmachine => get_standard_option('pve-qemu-machine', {
 596         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 597             ." for snapshots.",
 598     }),
 599     runningcpu => {
 600         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 601             ." internally for snapshots.",
 602         optional => 1,
 603         type => 'string',
 604         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 605         format_description => 'QEMU -cpu parameter'
 606     },
 607     machine => get_standard_option('pve-qemu-machine'),
 608     arch => {
 609         description => "Virtual processor architecture. Defaults to the host.",
 610         optional => 1,
 611         type => 'string',
 612         enum => [qw(x86_64 aarch64)],
 613     },
 614     smbios1 => {
 615         description => "Specify SMBIOS type 1 fields.",
 616         type => 'string', format => 'pve-qm-smbios1',
 617         maxLength => 512,
 618         optional => 1,
 619     },
 620     protection => {
 621         optional => 1,
 622         type => 'boolean',
 623         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 624             ." remove disk operations.",
 625         default => 0,
 626     },
 627     bios => {
 628         optional => 1,
 629         type => 'string',
 630         enum => [ qw(seabios ovmf) ],
 631         description => "Select BIOS implementation.",
 632         default => 'seabios',
 633     },
 634     vmgenid => {
 635         type => 'string',
 636         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 637         format_description => 'UUID',
 638         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 639             ." to disable explicitly.",
 640         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 641             ." value identifier to the guest OS. This allows to notify the guest operating system"
 642             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 643             ." execution or creation from a template). The guest operating system notices the"
 644             ." change, and is then able to react as appropriate by marking its copies of"
 645             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 646             ."Note that auto-creation only works when done through API/CLI create or update methods"
 647             .", but not when manually editing the config file.",
 648         default => "1 (autogenerated)",
 649         optional => 1,
 650     },
 651     hookscript => {
 652         type => 'string',
 653         format => 'pve-volume-id',
 654         optional => 1,
 655         description => "Script that will be executed during various steps in the vms lifetime.",
 656     },
 657     ivshmem => {
 658         type => 'string',
 659         format => $ivshmem_fmt,
 660         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 661             ." the host.",
 662         optional => 1,
 663     },
 664     audio0 => {
 665         type => 'string',
 666         format => $audio_fmt,
 667         description => "Configure a audio device, useful in combination with QXL/Spice.",
 668         optional => 1
 669     },
 670     spice_enhancements => {
 671         type => 'string',
 672         format => $spice_enhancements_fmt,
 673         description => "Configure additional enhancements for SPICE.",
 674         optional => 1
 675     },
 676     tags => {
 677         type => 'string', format => 'pve-tag-list',
 678         description => 'Tags of the VM. This is only meta information.',
 679         optional => 1,
 680     },
 681     rng0 => {
 682         type => 'string',
 683         format => $rng_fmt,
 684         description => "Configure a VirtIO-based Random Number Generator.",
 685         optional => 1,
 686     },
 687 };
 688
 689 my $cicustom_fmt = {
 690     meta => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all meta data passed to the VM via"
 694             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 695         format => 'pve-volume-id',
 696         format_description => 'volume',
 697     },
 698     network => {
 699         type => 'string',
 700         optional => 1,
 701         description => 'Specify a custom file containing all network data passed to the VM via'
 702             .' cloud-init.',
 703         format => 'pve-volume-id',
 704         format_description => 'volume',
 705     },
 706     user => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all user data passed to the VM via'
 710             .' cloud-init.',
 711         format => 'pve-volume-id',
 712         format_description => 'volume',
 713     },
 714 };
 715 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 716
 717 my $confdesc_cloudinit = {
 718     citype => {
 719         optional => 1,
 720         type => 'string',
 721         description => 'Specifies the cloud-init configuration format. The default depends on the'
 722             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 723             .' and `configdrive2` for windows.',
 724         enum => ['configdrive2', 'nocloud', 'opennebula'],
 725     },
 726     ciuser => {
 727         optional => 1,
 728         type => 'string',
 729         description => "cloud-init: User name to change ssh keys and password for instead of the"
 730             ." image's configured default user.",
 731     },
 732     cipassword => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'cloud-init: Password to assign the user. Using this is generally not'
 736             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 737             .' support hashed passwords.',
 738     },
 739     cicustom => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'cloud-init: Specify custom files to replace the automatically generated'
 743             .' ones at start.',
 744         format => 'pve-qm-cicustom',
 745     },
 746     searchdomain => {
 747         optional => 1,
 748         type => 'string',
 749         description => "cloud-init: Sets DNS search domains for a container. Create will'
 750             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 751             .' are set.",
 752     },
 753     nameserver => {
 754         optional => 1,
 755         type => 'string', format => 'address-list',
 756         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 757             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 758             .' are set.",
 759     },
 760     sshkeys => {
 761         optional => 1,
 762         type => 'string',
 763         format => 'urlencoded',
 764         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 765     },
 766 };
 767
 768 # what about other qemu settings ?
 769 #cpu => 'string',
 770 #machine => 'string',
 771 #fda => 'file',
 772 #fdb => 'file',
 773 #mtdblock => 'file',
 774 #sd => 'file',
 775 #pflash => 'file',
 776 #snapshot => 'bool',
 777 #bootp => 'file',
 778 ##tftp => 'dir',
 779 ##smb => 'dir',
 780 #kernel => 'file',
 781 #append => 'string',
 782 #initrd => 'file',
 783 ##soundhw => 'string',
 784
 785 while (my ($k, $v) = each %$confdesc) {
 786     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 787 }
 788
 789 my $MAX_USB_DEVICES = 5;
 790 my $MAX_NETS = 32;
 791 my $MAX_SERIAL_PORTS = 4;
 792 my $MAX_PARALLEL_PORTS = 3;
 793 my $MAX_NUMA = 8;
 794
 795 my $numa_fmt = {
 796     cpus => {
 797         type => "string",
 798         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 799         description => "CPUs accessing this NUMA node.",
 800         format_description => "id[-id];...",
 801     },
 802     memory => {
 803         type => "number",
 804         description => "Amount of memory this NUMA node provides.",
 805         optional => 1,
 806     },
 807     hostnodes => {
 808         type => "string",
 809         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 810         description => "Host NUMA nodes to use.",
 811         format_description => "id[-id];...",
 812         optional => 1,
 813     },
 814     policy => {
 815         type => 'string',
 816         enum => [qw(preferred bind interleave)],
 817         description => "NUMA allocation policy.",
 818         optional => 1,
 819     },
 820 };
 821 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 822 my $numadesc = {
 823     optional => 1,
 824     type => 'string', format => $numa_fmt,
 825     description => "NUMA topology.",
 826 };
 827 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 828
 829 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 830     $confdesc->{"numa$i"} = $numadesc;
 831 }
 832
 833 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 834                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 835                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 836 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 837
 838 my $net_fmt_bridge_descr = <<__EOD__;
 839 Bridge to attach the network device to. The Proxmox VE standard bridge
 840 is called 'vmbr0'.
 841
 842 If you do not specify a bridge, we create a kvm user (NATed) network
 843 device, which provides DHCP and DNS services. The following addresses
 844 are used:
 845
 846  10.0.2.2   Gateway
 847  10.0.2.3   DNS Server
 848  10.0.2.4   SMB Server
 849
 850 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 851 __EOD__
 852
 853 my $net_fmt = {
 854     macaddr  => get_standard_option('mac-addr', {
 855         description => "MAC address. That address must be unique withing your network. This is"
 856             ." automatically generated if not specified.",
 857     }),
 858     model => {
 859         type => 'string',
 860         description => "Network Card Model. The 'virtio' model provides the best performance with"
 861             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 862             ." best to use 'e1000'.",
 863         enum => $nic_model_list,
 864         default_key => 1,
 865     },
 866     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 867     bridge => {
 868         type => 'string',
 869         description => $net_fmt_bridge_descr,
 870         format_description => 'bridge',
 871         pattern => '[-_.\w\d]+',
 872         optional => 1,
 873     },
 874     queues => {
 875         type => 'integer',
 876         minimum => 0, maximum => 16,
 877         description => 'Number of packet queues to be used on the device.',
 878         optional => 1,
 879     },
 880     rate => {
 881         type => 'number',
 882         minimum => 0,
 883         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 884         optional => 1,
 885     },
 886     tag => {
 887         type => 'integer',
 888         minimum => 1, maximum => 4094,
 889         description => 'VLAN tag to apply to packets on this interface.',
 890         optional => 1,
 891     },
 892     trunks => {
 893         type => 'string',
 894         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 895         description => 'VLAN trunks to pass through this interface.',
 896         format_description => 'vlanid[;vlanid...]',
 897         optional => 1,
 898     },
 899     firewall => {
 900         type => 'boolean',
 901         description => 'Whether this interface should be protected by the firewall.',
 902         optional => 1,
 903     },
 904     link_down => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be disconnected (like pulling the plug).',
 907         optional => 1,
 908     },
 909     mtu => {
 910         type => 'integer',
 911         minimum => 1, maximum => 65520,
 912         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 913         optional => 1,
 914     },
 915 };
 916
 917 my $netdesc = {
 918     optional => 1,
 919     type => 'string', format => $net_fmt,
 920     description => "Specify network devices.",
 921 };
 922
 923 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 924
 925 my $ipconfig_fmt = {
 926     ip => {
 927         type => 'string',
 928         format => 'pve-ipv4-config',
 929         format_description => 'IPv4Format/CIDR',
 930         description => 'IPv4 address in CIDR format.',
 931         optional => 1,
 932         default => 'dhcp',
 933     },
 934     gw => {
 935         type => 'string',
 936         format => 'ipv4',
 937         format_description => 'GatewayIPv4',
 938         description => 'Default gateway for IPv4 traffic.',
 939         optional => 1,
 940         requires => 'ip',
 941     },
 942     ip6 => {
 943         type => 'string',
 944         format => 'pve-ipv6-config',
 945         format_description => 'IPv6Format/CIDR',
 946         description => 'IPv6 address in CIDR format.',
 947         optional => 1,
 948         default => 'dhcp',
 949     },
 950     gw6 => {
 951         type => 'string',
 952         format => 'ipv6',
 953         format_description => 'GatewayIPv6',
 954         description => 'Default gateway for IPv6 traffic.',
 955         optional => 1,
 956         requires => 'ip6',
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 960 my $ipconfigdesc = {
 961     optional => 1,
 962     type => 'string', format => 'pve-qm-ipconfig',
 963     description => <<'EODESCR',
 964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 965
 966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 967
 968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 969 gateway should be provided.
 970 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 971 cloud-init 19.4 or newer.
 972
 973 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 974 dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $usb_fmt = {
1006     host => {
1007         default_key => 1,
1008         type => 'string', format => 'pve-qm-usb-device',
1009         format_description => 'HOSTUSBDEVICE|spice',
1010         description => <<EODESCR,
1011 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1012
1013  'bus-port(.port)*' (decimal numbers) or
1014  'vendor_id:product_id' (hexadeciaml numbers) or
1015  'spice'
1016
1017 You can use the 'lsusb -t' command to list existing usb devices.
1018
1019 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1020 machines - use with special care.
1021
1022 The value 'spice' can be used to add a usb redirection devices for spice.
1023 EODESCR
1024     },
1025     usb3 => {
1026         optional => 1,
1027         type => 'boolean',
1028         description => "Specifies whether if given host option is a USB3 device or port.",
1029         default => 0,
1030     },
1031 };
1032
1033 my $usbdesc = {
1034     optional => 1,
1035     type => 'string', format => $usb_fmt,
1036     description => "Configure an USB device (n is 0 to 4).",
1037 };
1038 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1039
1040 my $serialdesc = {
1041         optional => 1,
1042         type => 'string',
1043         pattern => '(/dev/.+|socket)',
1044         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1045         verbose_description =>  <<EODESCR,
1046 Create a serial device inside the VM (n is 0 to 3), and pass through a
1047 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1048 host side (use 'qm terminal' to open a terminal connection).
1049
1050 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1051 use with special care.
1052
1053 CAUTION: Experimental! User reported problems with this option.
1054 EODESCR
1055 };
1056
1057 my $paralleldesc= {
1058         optional => 1,
1059         type => 'string',
1060         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1061         description =>  "Map host parallel devices (n is 0 to 2).",
1062         verbose_description =>  <<EODESCR,
1063 Map host parallel devices (n is 0 to 2).
1064
1065 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1066 machines - use with special care.
1067
1068 CAUTION: Experimental! User reported problems with this option.
1069 EODESCR
1070 };
1071
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1073     $confdesc->{"parallel$i"} = $paralleldesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1077     $confdesc->{"serial$i"} = $serialdesc;
1078 }
1079
1080 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1081     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1082 }
1083
1084 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1085     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1086 }
1087
1088 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1089     $confdesc->{"usb$i"} = $usbdesc;
1090 }
1091
1092 my $boot_fmt = {
1093     legacy => {
1094         optional => 1,
1095         default_key => 1,
1096         type => 'string',
1097         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1098                      . " Deprecated, use 'order=' instead.",
1099         pattern => '[acdn]{1,4}',
1100         format_description => "[acdn]{1,4}",
1101
1102         # note: this is also the fallback if boot: is not given at all
1103         default => 'cdn',
1104     },
1105     order => {
1106         optional => 1,
1107         type => 'string',
1108         format => 'pve-qm-bootdev-list',
1109         format_description => "device[;device...]",
1110         description => <<EODESC,
1111 The guest will attempt to boot from devices in the order they appear here.
1112
1113 Disks, optical drives and passed-through storage USB devices will be directly
1114 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1115 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1116
1117 Note that only devices in this list will be marked as bootable and thus loaded
1118 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1119 (e.g. software-raid), you need to specify all of them here.
1120
1121 Overrides the deprecated 'legacy=[acdn]*' value when given.
1122 EODESC
1123     },
1124 };
1125 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1126
1127 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1128 sub verify_bootdev {
1129     my ($dev, $noerr) = @_;
1130
1131     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1132
1133     my $check = sub {
1134         my ($base) = @_;
1135         return 0 if $dev !~ m/^$base\d+$/;
1136         return 0 if !$confdesc->{$dev};
1137         return 1;
1138     };
1139
1140     return $dev if $check->("net");
1141     return $dev if $check->("usb");
1142     return $dev if $check->("hostpci");
1143
1144     return if $noerr;
1145     die "invalid boot device '$dev'\n";
1146 }
1147
1148 sub print_bootorder {
1149     my ($devs) = @_;
1150     return "" if !@$devs;
1151     my $data = { order => join(';', @$devs) };
1152     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1153 }
1154
1155 my $kvm_api_version = 0;
1156
1157 sub kvm_version {
1158     return $kvm_api_version if $kvm_api_version;
1159
1160     open my $fh, '<', '/dev/kvm' or return;
1161
1162     # 0xae00 => KVM_GET_API_VERSION
1163     $kvm_api_version = ioctl($fh, 0xae00, 0);
1164     close($fh);
1165
1166     return $kvm_api_version;
1167 }
1168
1169 my $kvm_user_version = {};
1170 my $kvm_mtime = {};
1171
1172 sub kvm_user_version {
1173     my ($binary) = @_;
1174
1175     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1176     my $st = stat($binary);
1177
1178     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180         $cachedmtime == $st->mtime;
1181
1182     $kvm_user_version->{$binary} = 'unknown';
1183     $kvm_mtime->{$binary} = $st->mtime;
1184
1185     my $code = sub {
1186         my $line = shift;
1187         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188             $kvm_user_version->{$binary} = $2;
1189         }
1190     };
1191
1192     eval { run_command([$binary, '--version'], outfunc => $code); };
1193     warn $@ if $@;
1194
1195     return $kvm_user_version->{$binary};
1196
1197 }
1198 my sub extract_version {
1199     my ($machine_type, $version) = @_;
1200     $version = kvm_user_version() if !defined($version);
1201     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1202 }
1203
1204 sub kernel_has_vhost_net {
1205     return -c '/dev/vhost-net';
1206 }
1207
1208 sub option_exists {
1209     my $key = shift;
1210     return defined($confdesc->{$key});
1211 }
1212
1213 my $cdrom_path;
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241      if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1286             if !$vtype;
1287         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1288         verify_media_type($opt, $vtype, $drive->{media});
1289         $drive->{file} = $volid;
1290     }
1291
1292     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1293 }
1294
1295 sub parse_hotplug_features {
1296     my ($data) = @_;
1297
1298     my $res = {};
1299
1300     return $res if $data eq '0';
1301
1302     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1303
1304     foreach my $feature (PVE::Tools::split_list($data)) {
1305         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306             $res->{$1} = 1;
1307         } else {
1308             die "invalid hotplug feature '$feature'\n";
1309         }
1310     }
1311     return $res;
1312 }
1313
1314 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1315 sub pve_verify_hotplug_features {
1316     my ($value, $noerr) = @_;
1317
1318     return $value if parse_hotplug_features($value);
1319
1320     return if $noerr;
1321
1322     die "unable to parse hotplug option\n";
1323 }
1324
1325 sub scsi_inquiry {
1326     my($fh, $noerr) = @_;
1327
1328     my $SG_IO = 0x2285;
1329     my $SG_GET_VERSION_NUM = 0x2282;
1330
1331     my $versionbuf = "\x00" x 8;
1332     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333     if (!$ret) {
1334         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335         return;
1336     }
1337     my $version = unpack("I", $versionbuf);
1338     if ($version < 30000) {
1339         die "scsi generic interface too old\n"  if !$noerr;
1340         return;
1341     }
1342
1343     my $buf = "\x00" x 36;
1344     my $sensebuf = "\x00" x 8;
1345     my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347     # see /usr/include/scsi/sg.h
1348     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351                       length($sensebuf), 0, length($buf), $buf,
1352                       $cmd, $sensebuf, 6000);
1353
1354     $ret = ioctl($fh, $SG_IO, $packet);
1355     if (!$ret) {
1356         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357         return;
1358     }
1359
1360     my @res = unpack($sg_io_hdr_t, $packet);
1361     if ($res[17] || $res[18]) {
1362         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363         return;
1364     }
1365
1366     my $res = {};
1367     (my $byte0, my $byte1, $res->{vendor},
1368      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370     $res->{removable} = $byte1 & 128 ? 1 : 0;
1371     $res->{type} = $byte0 & 31;
1372
1373     return $res;
1374 }
1375
1376 sub path_is_scsi {
1377     my ($path) = @_;
1378
1379     my $fh = IO::File->new("+<$path") || return;
1380     my $res = scsi_inquiry($fh, 1);
1381     close($fh);
1382
1383     return $res;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf, $arch) = @_;
1388
1389     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus;
1393     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1394         $usbbus = 'ehci';
1395     } else {
1396         $usbbus = 'uhci';
1397     }
1398
1399     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400 }
1401
1402 sub print_keyboarddevice_full {
1403     my ($conf, $arch, $machine) = @_;
1404
1405     return if $arch ne 'aarch64';
1406
1407     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408 }
1409
1410 my sub get_drive_id {
1411     my ($drive) = @_;
1412     return "$drive->{interface}$drive->{index}";
1413 }
1414
1415 sub print_drivedevice_full {
1416     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1417
1418     my $device = '';
1419     my $maxdev = 0;
1420
1421     my $drive_id = get_drive_id($drive);
1422     if ($drive->{interface} eq 'virtio') {
1423         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1424         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1425         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1426     } elsif ($drive->{interface} eq 'scsi') {
1427
1428         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1429         my $unit = $drive->{index} % $maxdev;
1430         my $devicetype = 'hd';
1431         my $path = '';
1432         if (drive_is_cdrom($drive)) {
1433             $devicetype = 'cd';
1434         } else {
1435             if ($drive->{file} =~ m|^/|) {
1436                 $path = $drive->{file};
1437                 if (my $info = path_is_scsi($path)) {
1438                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1439                         $devicetype = 'block';
1440                     } elsif ($info->{type} == 1) { # tape
1441                         $devicetype = 'generic';
1442                     }
1443                 }
1444             } else {
1445                  $path = PVE::Storage::path($storecfg, $drive->{file});
1446             }
1447
1448             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1449             my $version = extract_version($machine_type, kvm_user_version());
1450             if ($path =~ m/^iscsi\:\/\// &&
1451                !min_version($version, 4, 1)) {
1452                 $devicetype = 'generic';
1453             }
1454         }
1455
1456         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1458         } else {
1459             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1460                 .",lun=$drive->{index}";
1461         }
1462         $device .= ",drive=drive-$drive_id,id=$drive_id";
1463
1464         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1465             $device .= ",rotation_rate=1";
1466         }
1467         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1468
1469     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1470         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1471         my $controller = int($drive->{index} / $maxdev);
1472         my $unit = $drive->{index} % $maxdev;
1473         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1474
1475         $device = "ide-$devicetype";
1476         if ($drive->{interface} eq 'ide') {
1477             $device .= ",bus=ide.$controller,unit=$unit";
1478         } else {
1479             $device .= ",bus=ahci$controller.$unit";
1480         }
1481         $device .= ",drive=drive-$drive_id,id=$drive_id";
1482
1483         if ($devicetype eq 'hd') {
1484             if (my $model = $drive->{model}) {
1485                 $model = URI::Escape::uri_unescape($model);
1486                 $device .= ",model=$model";
1487             }
1488             if ($drive->{ssd}) {
1489                 $device .= ",rotation_rate=1";
1490             }
1491         }
1492         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1493     } elsif ($drive->{interface} eq 'usb') {
1494         die "implement me";
1495         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1496     } else {
1497         die "unsupported interface type";
1498     }
1499
1500     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1501
1502     if (my $serial = $drive->{serial}) {
1503         $serial = URI::Escape::uri_unescape($serial);
1504         $device .= ",serial=$serial";
1505     }
1506
1507
1508     return $device;
1509 }
1510
1511 sub get_initiator_name {
1512     my $initiator;
1513
1514     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1515     while (defined(my $line = <$fh>)) {
1516         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1517         $initiator = $1;
1518         last;
1519     }
1520     $fh->close();
1521
1522     return $initiator;
1523 }
1524
1525 sub print_drive_commandline_full {
1526     my ($storecfg, $vmid, $drive, $pbs_name) = @_;
1527
1528     my $path;
1529     my $volid = $drive->{file};
1530     my $format = $drive->{format};
1531     my $drive_id = get_drive_id($drive);
1532
1533     if (drive_is_cdrom($drive)) {
1534         $path = get_iso_path($storecfg, $vmid, $volid);
1535         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1536     } else {
1537         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1538         if ($storeid) {
1539             $path = PVE::Storage::path($storecfg, $volid);
1540             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1541             $format //= qemu_img_format($scfg, $volname);
1542         } else {
1543             $path = $volid;
1544             $format //= "raw";
1545         }
1546    }
1547
1548    my $is_rbd = $path =~ m/^rbd:/;
1549
1550     my $opts = '';
1551     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1552     foreach my $o (@qemu_drive_options) {
1553         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1554     }
1555
1556     # snapshot only accepts on|off
1557     if (defined($drive->{snapshot})) {
1558         my $v = $drive->{snapshot} ? 'on' : 'off';
1559         $opts .= ",snapshot=$v";
1560     }
1561
1562     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1563         my ($dir, $qmpname) = @$type;
1564         if (my $v = $drive->{"mbps$dir"}) {
1565             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1566         }
1567         if (my $v = $drive->{"mbps${dir}_max"}) {
1568             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1569         }
1570         if (my $v = $drive->{"bps${dir}_max_length"}) {
1571             $opts .= ",throttling.bps$qmpname-max-length=$v";
1572         }
1573         if (my $v = $drive->{"iops${dir}"}) {
1574             $opts .= ",throttling.iops$qmpname=$v";
1575         }
1576         if (my $v = $drive->{"iops${dir}_max"}) {
1577             $opts .= ",throttling.iops$qmpname-max=$v";
1578         }
1579         if (my $v = $drive->{"iops${dir}_max_length"}) {
1580             $opts .= ",throttling.iops$qmpname-max-length=$v";
1581         }
1582     }
1583
1584     if ($pbs_name) {
1585         $format = "rbd" if $is_rbd;
1586         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1587             if !$format;
1588         $opts .= ",format=alloc-track,file.driver=$format";
1589     } elsif ($format) {
1590         $opts .= ",format=$format";
1591     }
1592
1593     my $cache_direct = 0;
1594
1595     if (my $cache = $drive->{cache}) {
1596         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1597     } elsif (!drive_is_cdrom($drive)) {
1598         $opts .= ",cache=none";
1599         $cache_direct = 1;
1600     }
1601
1602     # aio native works only with O_DIRECT
1603     if (!$drive->{aio}) {
1604         if($cache_direct) {
1605             $opts .= ",aio=native";
1606         } else {
1607             $opts .= ",aio=threads";
1608         }
1609     }
1610
1611     if (!drive_is_cdrom($drive)) {
1612         my $detectzeroes;
1613         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1614             $detectzeroes = 'off';
1615         } elsif ($drive->{discard}) {
1616             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1617         } else {
1618             # This used to be our default with discard not being specified:
1619             $detectzeroes = 'on';
1620         }
1621
1622         # note: 'detect-zeroes' works per blockdev and we want it to persist
1623         # after the alloc-track is removed, so put it on 'file' directly
1624         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1625         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1626     }
1627
1628     if ($pbs_name) {
1629         $opts .= ",backing=$pbs_name";
1630         $opts .= ",auto-remove=on";
1631     }
1632
1633     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1634     my $file_param = "file";
1635     if ($pbs_name) {
1636         # non-rbd drivers require the underlying file to be a seperate block
1637         # node, so add a second .file indirection
1638         $file_param .= ".file" if !$is_rbd;
1639         $file_param .= ".filename";
1640     }
1641     my $pathinfo = $path ? "$file_param=$path," : '';
1642
1643     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1644 }
1645
1646 sub print_pbs_blockdev {
1647     my ($pbs_conf, $pbs_name) = @_;
1648     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1649     $blockdev .= ",repository=$pbs_conf->{repository}";
1650     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1651     $blockdev .= ",archive=$pbs_conf->{archive}";
1652     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1653     return $blockdev;
1654 }
1655
1656 sub print_netdevice_full {
1657     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1658
1659     my $device = $net->{model};
1660     if ($net->{model} eq 'virtio') {
1661          $device = 'virtio-net-pci';
1662      };
1663
1664     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1665     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1666     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1667         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1668         # and out of each queue plus one config interrupt and control vector queue
1669         my $vectors = $net->{queues} * 2 + 2;
1670         $tmpstr .= ",vectors=$vectors,mq=on";
1671     }
1672     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1673
1674     if (my $mtu = $net->{mtu}) {
1675         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1676             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1677             if ($mtu == 1) {
1678                  $mtu = $bridge_mtu;
1679             } elsif ($mtu < 576) {
1680                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1681             } elsif ($mtu > $bridge_mtu) {
1682                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1683             }
1684             $tmpstr .= ",host_mtu=$mtu";
1685         } else {
1686             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1687         }
1688     }
1689
1690     if ($use_old_bios_files) {
1691         my $romfile;
1692         if ($device eq 'virtio-net-pci') {
1693             $romfile = 'pxe-virtio.rom';
1694         } elsif ($device eq 'e1000') {
1695             $romfile = 'pxe-e1000.rom';
1696         } elsif ($device eq 'ne2k') {
1697             $romfile = 'pxe-ne2k_pci.rom';
1698         } elsif ($device eq 'pcnet') {
1699             $romfile = 'pxe-pcnet.rom';
1700         } elsif ($device eq 'rtl8139') {
1701             $romfile = 'pxe-rtl8139.rom';
1702         }
1703         $tmpstr .= ",romfile=$romfile" if $romfile;
1704     }
1705
1706     return $tmpstr;
1707 }
1708
1709 sub print_netdev_full {
1710     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1711
1712     my $i = '';
1713     if ($netid =~ m/^net(\d+)$/) {
1714         $i = int($1);
1715     }
1716
1717     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1718
1719     my $ifname = "tap${vmid}i$i";
1720
1721     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1722     die "interface name '$ifname' is too long (max 15 character)\n"
1723         if length($ifname) >= 16;
1724
1725     my $vhostparam = '';
1726     if (is_native($arch)) {
1727         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1728     }
1729
1730     my $vmname = $conf->{name} || "vm$vmid";
1731
1732     my $netdev = "";
1733     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1734
1735     if ($net->{bridge}) {
1736         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1737             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1738     } else {
1739         $netdev = "type=user,id=$netid,hostname=$vmname";
1740     }
1741
1742     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1743
1744     return $netdev;
1745 }
1746
1747 my $vga_map = {
1748     'cirrus' => 'cirrus-vga',
1749     'std' => 'VGA',
1750     'vmware' => 'vmware-svga',
1751     'virtio' => 'virtio-vga',
1752 };
1753
1754 sub print_vga_device {
1755     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1756
1757     my $type = $vga_map->{$vga->{type}};
1758     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1759         $type = 'virtio-gpu';
1760     }
1761     my $vgamem_mb = $vga->{memory};
1762
1763     my $max_outputs = '';
1764     if ($qxlnum) {
1765         $type = $id ? 'qxl' : 'qxl-vga';
1766
1767         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1768             # set max outputs so linux can have up to 4 qxl displays with one device
1769             if (min_version($machine_version, 4, 1)) {
1770                 $max_outputs = ",max_outputs=4";
1771             }
1772         }
1773     }
1774
1775     die "no devicetype for $vga->{type}\n" if !$type;
1776
1777     my $memory = "";
1778     if ($vgamem_mb) {
1779         if ($vga->{type} eq 'virtio') {
1780             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1781             $memory = ",max_hostmem=$bytes";
1782         } elsif ($qxlnum) {
1783             # from https://www.spice-space.org/multiple-monitors.html
1784             $memory = ",vgamem_mb=$vga->{memory}";
1785             my $ram = $vgamem_mb * 4;
1786             my $vram = $vgamem_mb * 2;
1787             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1788         } else {
1789             $memory = ",vgamem_mb=$vga->{memory}";
1790         }
1791     } elsif ($qxlnum && $id) {
1792         $memory = ",ram_size=67108864,vram_size=33554432";
1793     }
1794
1795     my $edidoff = "";
1796     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1797         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1798     }
1799
1800     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1801     my $vgaid = "vga" . ($id // '');
1802     my $pciaddr;
1803
1804     if ($q35 && $vgaid eq 'vga') {
1805         # the first display uses pcie.0 bus on q35 machines
1806         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1807     } else {
1808         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1809     }
1810
1811     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1812 }
1813
1814 sub parse_number_sets {
1815     my ($set) = @_;
1816     my $res = [];
1817     foreach my $part (split(/;/, $set)) {
1818         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1819             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1820             push @$res, [ $1, $2 ];
1821         } else {
1822             die "invalid range: $part\n";
1823         }
1824     }
1825     return $res;
1826 }
1827
1828 sub parse_numa {
1829     my ($data) = @_;
1830
1831     my $res = parse_property_string($numa_fmt, $data);
1832     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1833     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1834     return $res;
1835 }
1836
1837 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1838 sub parse_net {
1839     my ($data) = @_;
1840
1841     my $res = eval { parse_property_string($net_fmt, $data) };
1842     if ($@) {
1843         warn $@;
1844         return;
1845     }
1846     if (!defined($res->{macaddr})) {
1847         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1848         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1849     }
1850     return $res;
1851 }
1852
1853 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1854 sub parse_ipconfig {
1855     my ($data) = @_;
1856
1857     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1858     if ($@) {
1859         warn $@;
1860         return;
1861     }
1862
1863     if ($res->{gw} && !$res->{ip}) {
1864         warn 'gateway specified without specifying an IP address';
1865         return;
1866     }
1867     if ($res->{gw6} && !$res->{ip6}) {
1868         warn 'IPv6 gateway specified without specifying an IPv6 address';
1869         return;
1870     }
1871     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1872         warn 'gateway specified together with DHCP';
1873         return;
1874     }
1875     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1876         # gw6 + auto/dhcp
1877         warn "IPv6 gateway specified together with $res->{ip6} address";
1878         return;
1879     }
1880
1881     if (!$res->{ip} && !$res->{ip6}) {
1882         return { ip => 'dhcp', ip6 => 'dhcp' };
1883     }
1884
1885     return $res;
1886 }
1887
1888 sub print_net {
1889     my $net = shift;
1890
1891     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1892 }
1893
1894 sub add_random_macs {
1895     my ($settings) = @_;
1896
1897     foreach my $opt (keys %$settings) {
1898         next if $opt !~ m/^net(\d+)$/;
1899         my $net = parse_net($settings->{$opt});
1900         next if !$net;
1901         $settings->{$opt} = print_net($net);
1902     }
1903 }
1904
1905 sub vm_is_volid_owner {
1906     my ($storecfg, $vmid, $volid) = @_;
1907
1908     if ($volid !~  m|^/|) {
1909         my ($path, $owner);
1910         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1911         if ($owner && ($owner == $vmid)) {
1912             return 1;
1913         }
1914     }
1915
1916     return;
1917 }
1918
1919 sub vmconfig_register_unused_drive {
1920     my ($storecfg, $vmid, $conf, $drive) = @_;
1921
1922     if (drive_is_cloudinit($drive)) {
1923         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1924         warn $@ if $@;
1925     } elsif (!drive_is_cdrom($drive)) {
1926         my $volid = $drive->{file};
1927         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1928             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1929         }
1930     }
1931 }
1932
1933 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1934 my $smbios1_fmt = {
1935     uuid => {
1936         type => 'string',
1937         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1938         format_description => 'UUID',
1939         description => "Set SMBIOS1 UUID.",
1940         optional => 1,
1941     },
1942     version => {
1943         type => 'string',
1944         pattern => '[A-Za-z0-9+\/]+={0,2}',
1945         format_description => 'Base64 encoded string',
1946         description => "Set SMBIOS1 version.",
1947         optional => 1,
1948     },
1949     serial => {
1950         type => 'string',
1951         pattern => '[A-Za-z0-9+\/]+={0,2}',
1952         format_description => 'Base64 encoded string',
1953         description => "Set SMBIOS1 serial number.",
1954         optional => 1,
1955     },
1956     manufacturer => {
1957         type => 'string',
1958         pattern => '[A-Za-z0-9+\/]+={0,2}',
1959         format_description => 'Base64 encoded string',
1960         description => "Set SMBIOS1 manufacturer.",
1961         optional => 1,
1962     },
1963     product => {
1964         type => 'string',
1965         pattern => '[A-Za-z0-9+\/]+={0,2}',
1966         format_description => 'Base64 encoded string',
1967         description => "Set SMBIOS1 product ID.",
1968         optional => 1,
1969     },
1970     sku => {
1971         type => 'string',
1972         pattern => '[A-Za-z0-9+\/]+={0,2}',
1973         format_description => 'Base64 encoded string',
1974         description => "Set SMBIOS1 SKU string.",
1975         optional => 1,
1976     },
1977     family => {
1978         type => 'string',
1979         pattern => '[A-Za-z0-9+\/]+={0,2}',
1980         format_description => 'Base64 encoded string',
1981         description => "Set SMBIOS1 family string.",
1982         optional => 1,
1983     },
1984     base64 => {
1985         type => 'boolean',
1986         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1987         optional => 1,
1988     },
1989 };
1990
1991 sub parse_smbios1 {
1992     my ($data) = @_;
1993
1994     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1995     warn $@ if $@;
1996     return $res;
1997 }
1998
1999 sub print_smbios1 {
2000     my ($smbios1) = @_;
2001     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2002 }
2003
2004 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2005
2006 sub parse_watchdog {
2007     my ($value) = @_;
2008
2009     return if !$value;
2010
2011     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2012     warn $@ if $@;
2013     return $res;
2014 }
2015
2016 sub parse_guest_agent {
2017     my ($conf) = @_;
2018
2019     return {} if !defined($conf->{agent});
2020
2021     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2022     warn $@ if $@;
2023
2024     # if the agent is disabled ignore the other potentially set properties
2025     return {} if !$res->{enabled};
2026     return $res;
2027 }
2028
2029 sub get_qga_key {
2030     my ($conf, $key) = @_;
2031     return undef if !defined($conf->{agent});
2032
2033     my $agent = parse_guest_agent($conf);
2034     return $agent->{$key};
2035 }
2036
2037 sub parse_vga {
2038     my ($value) = @_;
2039
2040     return {} if !$value;
2041     my $res = eval { parse_property_string($vga_fmt, $value) };
2042     warn $@ if $@;
2043     return $res;
2044 }
2045
2046 sub parse_rng {
2047     my ($value) = @_;
2048
2049     return if !$value;
2050
2051     my $res = eval { parse_property_string($rng_fmt, $value) };
2052     warn $@ if $@;
2053     return $res;
2054 }
2055
2056 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2057 sub verify_usb_device {
2058     my ($value, $noerr) = @_;
2059
2060     return $value if parse_usb_device($value);
2061
2062     return if $noerr;
2063
2064     die "unable to parse usb device\n";
2065 }
2066
2067 # add JSON properties for create and set function
2068 sub json_config_properties {
2069     my $prop = shift;
2070
2071     foreach my $opt (keys %$confdesc) {
2072         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2073             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2074         $prop->{$opt} = $confdesc->{$opt};
2075     }
2076
2077     return $prop;
2078 }
2079
2080 # return copy of $confdesc_cloudinit to generate documentation
2081 sub cloudinit_config_properties {
2082
2083     return dclone($confdesc_cloudinit);
2084 }
2085
2086 sub check_type {
2087     my ($key, $value) = @_;
2088
2089     die "unknown setting '$key'\n" if !$confdesc->{$key};
2090
2091     my $type = $confdesc->{$key}->{type};
2092
2093     if (!defined($value)) {
2094         die "got undefined value\n";
2095     }
2096
2097     if ($value =~ m/[\n\r]/) {
2098         die "property contains a line feed\n";
2099     }
2100
2101     if ($type eq 'boolean') {
2102         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2103         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2104         die "type check ('boolean') failed - got '$value'\n";
2105     } elsif ($type eq 'integer') {
2106         return int($1) if $value =~ m/^(\d+)$/;
2107         die "type check ('integer') failed - got '$value'\n";
2108     } elsif ($type eq 'number') {
2109         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2110         die "type check ('number') failed - got '$value'\n";
2111     } elsif ($type eq 'string') {
2112         if (my $fmt = $confdesc->{$key}->{format}) {
2113             PVE::JSONSchema::check_format($fmt, $value);
2114             return $value;
2115         }
2116         $value =~ s/^\"(.*)\"$/$1/;
2117         return $value;
2118     } else {
2119         die "internal error"
2120     }
2121 }
2122
2123 sub destroy_vm {
2124     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2125
2126     my $conf = PVE::QemuConfig->load_config($vmid);
2127
2128     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2129
2130     if ($conf->{template}) {
2131         # check if any base image is still used by a linked clone
2132         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2133                 my ($ds, $drive) = @_;
2134                 return if drive_is_cdrom($drive);
2135
2136                 my $volid = $drive->{file};
2137                 return if !$volid || $volid =~ m|^/|;
2138
2139                 die "base volume '$volid' is still in use by linked cloned\n"
2140                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2141
2142         });
2143     }
2144
2145     my $remove_owned_drive = sub {
2146         my ($ds, $drive) = @_;
2147         return if drive_is_cdrom($drive, 1);
2148
2149         my $volid = $drive->{file};
2150         return if !$volid || $volid =~ m|^/|;
2151
2152         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2153         return if !$path || !$owner || ($owner != $vmid);
2154
2155         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2156         warn "Could not remove disk '$volid', check manually: $@" if $@;
2157     };
2158
2159     # only remove disks owned by this VM (referenced in the config)
2160     my $include_opts = {
2161         include_unused => 1,
2162         extra_keys => ['vmstate'],
2163     };
2164     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2165
2166     for my $snap (values %{$conf->{snapshots}}) {
2167         next if !defined($snap->{vmstate});
2168         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2169         next if !defined($drive);
2170         $remove_owned_drive->('vmstate', $drive);
2171     }
2172
2173     if ($purge_unreferenced) { # also remove unreferenced disk
2174         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2175         PVE::Storage::foreach_volid($vmdisks, sub {
2176             my ($volid, $sid, $volname, $d) = @_;
2177             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2178             warn $@ if $@;
2179         });
2180     }
2181
2182     if (defined $replacement_conf) {
2183         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2184     } else {
2185         PVE::QemuConfig->destroy_config($vmid);
2186     }
2187 }
2188
2189 sub parse_vm_config {
2190     my ($filename, $raw) = @_;
2191
2192     return if !defined($raw);
2193
2194     my $res = {
2195         digest => Digest::SHA::sha1_hex($raw),
2196         snapshots => {},
2197         pending => {},
2198     };
2199
2200     $filename =~ m|/qemu-server/(\d+)\.conf$|
2201         || die "got strange filename '$filename'";
2202
2203     my $vmid = $1;
2204
2205     my $conf = $res;
2206     my $descr;
2207     my $section = '';
2208
2209     my @lines = split(/\n/, $raw);
2210     foreach my $line (@lines) {
2211         next if $line =~ m/^\s*$/;
2212
2213         if ($line =~ m/^\[PENDING\]\s*$/i) {
2214             $section = 'pending';
2215             if (defined($descr)) {
2216                 $descr =~ s/\s+$//;
2217                 $conf->{description} = $descr;
2218             }
2219             $descr = undef;
2220             $conf = $res->{$section} = {};
2221             next;
2222
2223         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2224             $section = $1;
2225             if (defined($descr)) {
2226                 $descr =~ s/\s+$//;
2227                 $conf->{description} = $descr;
2228             }
2229             $descr = undef;
2230             $conf = $res->{snapshots}->{$section} = {};
2231             next;
2232         }
2233
2234         if ($line =~ m/^\#(.*)\s*$/) {
2235             $descr = '' if !defined($descr);
2236             $descr .= PVE::Tools::decode_text($1) . "\n";
2237             next;
2238         }
2239
2240         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2241             $descr = '' if !defined($descr);
2242             $descr .= PVE::Tools::decode_text($2);
2243         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2244             $conf->{snapstate} = $1;
2245         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2246             my $key = $1;
2247             my $value = $2;
2248             $conf->{$key} = $value;
2249         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2250             my $value = $1;
2251             if ($section eq 'pending') {
2252                 $conf->{delete} = $value; # we parse this later
2253             } else {
2254                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2255             }
2256         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2257             my $key = $1;
2258             my $value = $2;
2259             eval { $value = check_type($key, $value); };
2260             if ($@) {
2261                 warn "vm $vmid - unable to parse value of '$key' - $@";
2262             } else {
2263                 $key = 'ide2' if $key eq 'cdrom';
2264                 my $fmt = $confdesc->{$key}->{format};
2265                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2266                     my $v = parse_drive($key, $value);
2267                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2268                         $v->{file} = $volid;
2269                         $value = print_drive($v);
2270                     } else {
2271                         warn "vm $vmid - unable to parse value of '$key'\n";
2272                         next;
2273                     }
2274                 }
2275
2276                 $conf->{$key} = $value;
2277             }
2278         } else {
2279             warn "vm $vmid - unable to parse config: $line\n";
2280         }
2281     }
2282
2283     if (defined($descr)) {
2284         $descr =~ s/\s+$//;
2285         $conf->{description} = $descr;
2286     }
2287     delete $res->{snapstate}; # just to be sure
2288
2289     return $res;
2290 }
2291
2292 sub write_vm_config {
2293     my ($filename, $conf) = @_;
2294
2295     delete $conf->{snapstate}; # just to be sure
2296
2297     if ($conf->{cdrom}) {
2298         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2299         $conf->{ide2} = $conf->{cdrom};
2300         delete $conf->{cdrom};
2301     }
2302
2303     # we do not use 'smp' any longer
2304     if ($conf->{sockets}) {
2305         delete $conf->{smp};
2306     } elsif ($conf->{smp}) {
2307         $conf->{sockets} = $conf->{smp};
2308         delete $conf->{cores};
2309         delete $conf->{smp};
2310     }
2311
2312     my $used_volids = {};
2313
2314     my $cleanup_config = sub {
2315         my ($cref, $pending, $snapname) = @_;
2316
2317         foreach my $key (keys %$cref) {
2318             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2319                 $key eq 'snapstate' || $key eq 'pending';
2320             my $value = $cref->{$key};
2321             if ($key eq 'delete') {
2322                 die "propertry 'delete' is only allowed in [PENDING]\n"
2323                     if !$pending;
2324                 # fixme: check syntax?
2325                 next;
2326             }
2327             eval { $value = check_type($key, $value); };
2328             die "unable to parse value of '$key' - $@" if $@;
2329
2330             $cref->{$key} = $value;
2331
2332             if (!$snapname && is_valid_drivename($key)) {
2333                 my $drive = parse_drive($key, $value);
2334                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2335             }
2336         }
2337     };
2338
2339     &$cleanup_config($conf);
2340
2341     &$cleanup_config($conf->{pending}, 1);
2342
2343     foreach my $snapname (keys %{$conf->{snapshots}}) {
2344         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2345         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2346     }
2347
2348     # remove 'unusedX' settings if we re-add a volume
2349     foreach my $key (keys %$conf) {
2350         my $value = $conf->{$key};
2351         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2352             delete $conf->{$key};
2353         }
2354     }
2355
2356     my $generate_raw_config = sub {
2357         my ($conf, $pending) = @_;
2358
2359         my $raw = '';
2360
2361         # add description as comment to top of file
2362         if (defined(my $descr = $conf->{description})) {
2363             if ($descr) {
2364                 foreach my $cl (split(/\n/, $descr)) {
2365                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2366                 }
2367             } else {
2368                 $raw .= "#\n" if $pending;
2369             }
2370         }
2371
2372         foreach my $key (sort keys %$conf) {
2373             next if $key =~ /^(digest|description|pending|snapshots)$/;
2374             $raw .= "$key: $conf->{$key}\n";
2375         }
2376         return $raw;
2377     };
2378
2379     my $raw = &$generate_raw_config($conf);
2380
2381     if (scalar(keys %{$conf->{pending}})){
2382         $raw .= "\n[PENDING]\n";
2383         $raw .= &$generate_raw_config($conf->{pending}, 1);
2384     }
2385
2386     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2387         $raw .= "\n[$snapname]\n";
2388         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2389     }
2390
2391     return $raw;
2392 }
2393
2394 sub load_defaults {
2395
2396     my $res = {};
2397
2398     # we use static defaults from our JSON schema configuration
2399     foreach my $key (keys %$confdesc) {
2400         if (defined(my $default = $confdesc->{$key}->{default})) {
2401             $res->{$key} = $default;
2402         }
2403     }
2404
2405     return $res;
2406 }
2407
2408 sub config_list {
2409     my $vmlist = PVE::Cluster::get_vmlist();
2410     my $res = {};
2411     return $res if !$vmlist || !$vmlist->{ids};
2412     my $ids = $vmlist->{ids};
2413     my $nodename = nodename();
2414
2415     foreach my $vmid (keys %$ids) {
2416         my $d = $ids->{$vmid};
2417         next if !$d->{node} || $d->{node} ne $nodename;
2418         next if !$d->{type} || $d->{type} ne 'qemu';
2419         $res->{$vmid}->{exists} = 1;
2420     }
2421     return $res;
2422 }
2423
2424 # test if VM uses local resources (to prevent migration)
2425 sub check_local_resources {
2426     my ($conf, $noerr) = @_;
2427
2428     my @loc_res = ();
2429
2430     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2431     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2432
2433     push @loc_res, "ivshmem" if $conf->{ivshmem};
2434
2435     foreach my $k (keys %$conf) {
2436         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2437         # sockets are safe: they will recreated be on the target side post-migrate
2438         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2439         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2440     }
2441
2442     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2443
2444     return \@loc_res;
2445 }
2446
2447 # check if used storages are available on all nodes (use by migrate)
2448 sub check_storage_availability {
2449     my ($storecfg, $conf, $node) = @_;
2450
2451     PVE::QemuConfig->foreach_volume($conf, sub {
2452         my ($ds, $drive) = @_;
2453
2454         my $volid = $drive->{file};
2455         return if !$volid;
2456
2457         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2458         return if !$sid;
2459
2460         # check if storage is available on both nodes
2461         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2462         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2463    });
2464 }
2465
2466 # list nodes where all VM images are available (used by has_feature API)
2467 sub shared_nodes {
2468     my ($conf, $storecfg) = @_;
2469
2470     my $nodelist = PVE::Cluster::get_nodelist();
2471     my $nodehash = { map { $_ => 1 } @$nodelist };
2472     my $nodename = nodename();
2473
2474     PVE::QemuConfig->foreach_volume($conf, sub {
2475         my ($ds, $drive) = @_;
2476
2477         my $volid = $drive->{file};
2478         return if !$volid;
2479
2480         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2481         if ($storeid) {
2482             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2483             if ($scfg->{disable}) {
2484                 $nodehash = {};
2485             } elsif (my $avail = $scfg->{nodes}) {
2486                 foreach my $node (keys %$nodehash) {
2487                     delete $nodehash->{$node} if !$avail->{$node};
2488                 }
2489             } elsif (!$scfg->{shared}) {
2490                 foreach my $node (keys %$nodehash) {
2491                     delete $nodehash->{$node} if $node ne $nodename
2492                 }
2493             }
2494         }
2495     });
2496
2497     return $nodehash
2498 }
2499
2500 sub check_local_storage_availability {
2501     my ($conf, $storecfg) = @_;
2502
2503     my $nodelist = PVE::Cluster::get_nodelist();
2504     my $nodehash = { map { $_ => {} } @$nodelist };
2505
2506     PVE::QemuConfig->foreach_volume($conf, sub {
2507         my ($ds, $drive) = @_;
2508
2509         my $volid = $drive->{file};
2510         return if !$volid;
2511
2512         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2513         if ($storeid) {
2514             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2515
2516             if ($scfg->{disable}) {
2517                 foreach my $node (keys %$nodehash) {
2518                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2519                 }
2520             } elsif (my $avail = $scfg->{nodes}) {
2521                 foreach my $node (keys %$nodehash) {
2522                     if (!$avail->{$node}) {
2523                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2524                     }
2525                 }
2526             }
2527         }
2528     });
2529
2530     foreach my $node (values %$nodehash) {
2531         if (my $unavail = $node->{unavailable_storages}) {
2532             $node->{unavailable_storages} = [ sort keys %$unavail ];
2533         }
2534     }
2535
2536     return $nodehash
2537 }
2538
2539 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2540 sub check_running {
2541     my ($vmid, $nocheck, $node) = @_;
2542
2543     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2544     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2545 }
2546
2547 sub vzlist {
2548
2549     my $vzlist = config_list();
2550
2551     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2552
2553     while (defined(my $de = $fd->read)) {
2554         next if $de !~ m/^(\d+)\.pid$/;
2555         my $vmid = $1;
2556         next if !defined($vzlist->{$vmid});
2557         if (my $pid = check_running($vmid)) {
2558             $vzlist->{$vmid}->{pid} = $pid;
2559         }
2560     }
2561
2562     return $vzlist;
2563 }
2564
2565 our $vmstatus_return_properties = {
2566     vmid => get_standard_option('pve-vmid'),
2567     status => {
2568         description => "Qemu process status.",
2569         type => 'string',
2570         enum => ['stopped', 'running'],
2571     },
2572     maxmem => {
2573         description => "Maximum memory in bytes.",
2574         type => 'integer',
2575         optional => 1,
2576         renderer => 'bytes',
2577     },
2578     maxdisk => {
2579         description => "Root disk size in bytes.",
2580         type => 'integer',
2581         optional => 1,
2582         renderer => 'bytes',
2583     },
2584     name => {
2585         description => "VM name.",
2586         type => 'string',
2587         optional => 1,
2588     },
2589     qmpstatus => {
2590         description => "Qemu QMP agent status.",
2591         type => 'string',
2592         optional => 1,
2593     },
2594     pid => {
2595         description => "PID of running qemu process.",
2596         type => 'integer',
2597         optional => 1,
2598     },
2599     uptime => {
2600         description => "Uptime.",
2601         type => 'integer',
2602         optional => 1,
2603         renderer => 'duration',
2604     },
2605     cpus => {
2606         description => "Maximum usable CPUs.",
2607         type => 'number',
2608         optional => 1,
2609     },
2610     lock => {
2611         description => "The current config lock, if any.",
2612         type => 'string',
2613         optional => 1,
2614     },
2615     tags => {
2616         description  => "The current configured tags, if any",
2617         type => 'string',
2618         optional => 1,
2619     },
2620     'running-machine' => {
2621         description => "The currently running machine type (if running).",
2622         type => 'string',
2623         optional => 1,
2624     },
2625     'running-qemu' => {
2626         description => "The currently running QEMU version (if running).",
2627         type => 'string',
2628         optional => 1,
2629     },
2630 };
2631
2632 my $last_proc_pid_stat;
2633
2634 # get VM status information
2635 # This must be fast and should not block ($full == false)
2636 # We only query KVM using QMP if $full == true (this can be slow)
2637 sub vmstatus {
2638     my ($opt_vmid, $full) = @_;
2639
2640     my $res = {};
2641
2642     my $storecfg = PVE::Storage::config();
2643
2644     my $list = vzlist();
2645     my $defaults = load_defaults();
2646
2647     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2648
2649     my $cpucount = $cpuinfo->{cpus} || 1;
2650
2651     foreach my $vmid (keys %$list) {
2652         next if $opt_vmid && ($vmid ne $opt_vmid);
2653
2654         my $conf = PVE::QemuConfig->load_config($vmid);
2655
2656         my $d = { vmid => int($vmid) };
2657         $d->{pid} = int($list->{$vmid}->{pid});
2658
2659         # fixme: better status?
2660         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2661
2662         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2663         if (defined($size)) {
2664             $d->{disk} = 0; # no info available
2665             $d->{maxdisk} = $size;
2666         } else {
2667             $d->{disk} = 0;
2668             $d->{maxdisk} = 0;
2669         }
2670
2671         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2672             * ($conf->{cores} || $defaults->{cores});
2673         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2674         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2675
2676         $d->{name} = $conf->{name} || "VM $vmid";
2677         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2678             : $defaults->{memory}*(1024*1024);
2679
2680         if ($conf->{balloon}) {
2681             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2682             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2683                 : $defaults->{shares};
2684         }
2685
2686         $d->{uptime} = 0;
2687         $d->{cpu} = 0;
2688         $d->{mem} = 0;
2689
2690         $d->{netout} = 0;
2691         $d->{netin} = 0;
2692
2693         $d->{diskread} = 0;
2694         $d->{diskwrite} = 0;
2695
2696         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2697
2698         $d->{serial} = 1 if conf_has_serial($conf);
2699         $d->{lock} = $conf->{lock} if $conf->{lock};
2700         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2701
2702         $res->{$vmid} = $d;
2703     }
2704
2705     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2706     foreach my $dev (keys %$netdev) {
2707         next if $dev !~ m/^tap([1-9]\d*)i/;
2708         my $vmid = $1;
2709         my $d = $res->{$vmid};
2710         next if !$d;
2711
2712         $d->{netout} += $netdev->{$dev}->{receive};
2713         $d->{netin} += $netdev->{$dev}->{transmit};
2714
2715         if ($full) {
2716             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2717             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2718         }
2719
2720     }
2721
2722     my $ctime = gettimeofday;
2723
2724     foreach my $vmid (keys %$list) {
2725
2726         my $d = $res->{$vmid};
2727         my $pid = $d->{pid};
2728         next if !$pid;
2729
2730         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2731         next if !$pstat; # not running
2732
2733         my $used = $pstat->{utime} + $pstat->{stime};
2734
2735         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2736
2737         if ($pstat->{vsize}) {
2738             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2739         }
2740
2741         my $old = $last_proc_pid_stat->{$pid};
2742         if (!$old) {
2743             $last_proc_pid_stat->{$pid} = {
2744                 time => $ctime,
2745                 used => $used,
2746                 cpu => 0,
2747             };
2748             next;
2749         }
2750
2751         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2752
2753         if ($dtime > 1000) {
2754             my $dutime = $used -  $old->{used};
2755
2756             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2757             $last_proc_pid_stat->{$pid} = {
2758                 time => $ctime,
2759                 used => $used,
2760                 cpu => $d->{cpu},
2761             };
2762         } else {
2763             $d->{cpu} = $old->{cpu};
2764         }
2765     }
2766
2767     return $res if !$full;
2768
2769     my $qmpclient = PVE::QMPClient->new();
2770
2771     my $ballooncb = sub {
2772         my ($vmid, $resp) = @_;
2773
2774         my $info = $resp->{'return'};
2775         return if !$info->{max_mem};
2776
2777         my $d = $res->{$vmid};
2778
2779         # use memory assigned to VM
2780         $d->{maxmem} = $info->{max_mem};
2781         $d->{balloon} = $info->{actual};
2782
2783         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2784             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2785             $d->{freemem} = $info->{free_mem};
2786         }
2787
2788         $d->{ballooninfo} = $info;
2789     };
2790
2791     my $blockstatscb = sub {
2792         my ($vmid, $resp) = @_;
2793         my $data = $resp->{'return'} || [];
2794         my $totalrdbytes = 0;
2795         my $totalwrbytes = 0;
2796
2797         for my $blockstat (@$data) {
2798             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2799             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2800
2801             $blockstat->{device} =~ s/drive-//;
2802             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2803         }
2804         $res->{$vmid}->{diskread} = $totalrdbytes;
2805         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2806     };
2807
2808     my $machinecb = sub {
2809         my ($vmid, $resp) = @_;
2810         my $data = $resp->{'return'} || [];
2811
2812         $res->{$vmid}->{'running-machine'} =
2813             PVE::QemuServer::Machine::current_from_query_machines($data);
2814     };
2815
2816     my $versioncb = sub {
2817         my ($vmid, $resp) = @_;
2818         my $data = $resp->{'return'} // {};
2819         my $version = 'unknown';
2820
2821         if (my $v = $data->{qemu}) {
2822             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2823         }
2824
2825         $res->{$vmid}->{'running-qemu'} = $version;
2826     };
2827
2828     my $statuscb = sub {
2829         my ($vmid, $resp) = @_;
2830
2831         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2832         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2833         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2834         # this fails if ballon driver is not loaded, so this must be
2835         # the last commnand (following command are aborted if this fails).
2836         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2837
2838         my $status = 'unknown';
2839         if (!defined($status = $resp->{'return'}->{status})) {
2840             warn "unable to get VM status\n";
2841             return;
2842         }
2843
2844         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2845     };
2846
2847     foreach my $vmid (keys %$list) {
2848         next if $opt_vmid && ($vmid ne $opt_vmid);
2849         next if !$res->{$vmid}->{pid}; # not running
2850         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2851     }
2852
2853     $qmpclient->queue_execute(undef, 2);
2854
2855     foreach my $vmid (keys %$list) {
2856         next if $opt_vmid && ($vmid ne $opt_vmid);
2857         next if !$res->{$vmid}->{pid}; #not running
2858
2859         # we can't use the $qmpclient since it might have already aborted on
2860         # 'query-balloon', but this might also fail for older versions...
2861         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2862         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2863     }
2864
2865     foreach my $vmid (keys %$list) {
2866         next if $opt_vmid && ($vmid ne $opt_vmid);
2867         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2868     }
2869
2870     return $res;
2871 }
2872
2873 sub conf_has_serial {
2874     my ($conf) = @_;
2875
2876     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2877         if ($conf->{"serial$i"}) {
2878             return 1;
2879         }
2880     }
2881
2882     return 0;
2883 }
2884
2885 sub conf_has_audio {
2886     my ($conf, $id) = @_;
2887
2888     $id //= 0;
2889     my $audio = $conf->{"audio$id"};
2890     return if !defined($audio);
2891
2892     my $audioproperties = parse_property_string($audio_fmt, $audio);
2893     my $audiodriver = $audioproperties->{driver} // 'spice';
2894
2895     return {
2896         dev => $audioproperties->{device},
2897         dev_id => "audiodev$id",
2898         backend => $audiodriver,
2899         backend_id => "$audiodriver-backend${id}",
2900     };
2901 }
2902
2903 sub audio_devs {
2904     my ($audio, $audiopciaddr, $machine_version) = @_;
2905
2906     my $devs = [];
2907
2908     my $id = $audio->{dev_id};
2909     my $audiodev = "";
2910     if (min_version($machine_version, 4, 2)) {
2911         $audiodev = ",audiodev=$audio->{backend_id}";
2912     }
2913
2914     if ($audio->{dev} eq 'AC97') {
2915         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2916     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2917         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2918         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2919         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2920     } else {
2921         die "unkown audio device '$audio->{dev}', implement me!";
2922     }
2923
2924     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2925
2926     return $devs;
2927 }
2928
2929 sub vga_conf_has_spice {
2930     my ($vga) = @_;
2931
2932     my $vgaconf = parse_vga($vga);
2933     my $vgatype = $vgaconf->{type};
2934     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2935
2936     return $1 || 1;
2937 }
2938
2939 sub is_native($) {
2940     my ($arch) = @_;
2941     return get_host_arch() eq $arch;
2942 }
2943
2944 sub get_vm_arch {
2945     my ($conf) = @_;
2946     return $conf->{arch} // get_host_arch();
2947 }
2948
2949 my $default_machines = {
2950     x86_64 => 'pc',
2951     aarch64 => 'virt',
2952 };
2953
2954 sub get_installed_machine_version {
2955     my ($kvmversion) = @_;
2956     $kvmversion = kvm_user_version() if !defined($kvmversion);
2957     $kvmversion =~ m/^(\d+\.\d+)/;
2958     return $1;
2959 }
2960
2961 sub windows_get_pinned_machine_version {
2962     my ($machine, $base_version, $kvmversion) = @_;
2963
2964     my $pin_version = $base_version;
2965     if (!defined($base_version) ||
2966         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2967     ) {
2968         $pin_version = get_installed_machine_version($kvmversion);
2969     }
2970     if (!$machine || $machine eq 'pc') {
2971         $machine = "pc-i440fx-$pin_version";
2972     } elsif ($machine eq 'q35') {
2973         $machine = "pc-q35-$pin_version";
2974     } elsif ($machine eq 'virt') {
2975         $machine = "virt-$pin_version";
2976     } else {
2977         warn "unknown machine type '$machine', not touching that!\n";
2978     }
2979
2980     return $machine;
2981 }
2982
2983 sub get_vm_machine {
2984     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2985
2986     my $machine = $forcemachine || $conf->{machine};
2987
2988     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2989         $kvmversion //= kvm_user_version();
2990         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2991         # layout which confuses windows quite a bit and may result in various regressions..
2992         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2993         if (windows_version($conf->{ostype})) {
2994             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2995         }
2996         $arch //= 'x86_64';
2997         $machine ||= $default_machines->{$arch};
2998         if ($add_pve_version) {
2999             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3000             $machine .= "+pve$pvever";
3001         }
3002     }
3003
3004     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3005         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3006         $machine = $1 if $is_pxe;
3007
3008         # for version-pinned machines that do not include a pve-version (e.g.
3009         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3010         $machine .= '+pve0';
3011
3012         $machine .= '.pxe' if $is_pxe;
3013     }
3014
3015     return $machine;
3016 }
3017
3018 sub get_ovmf_files($) {
3019     my ($arch) = @_;
3020
3021     my $ovmf = $OVMF->{$arch}
3022         or die "no OVMF images known for architecture '$arch'\n";
3023
3024     return @$ovmf;
3025 }
3026
3027 my $Arch2Qemu = {
3028     aarch64 => '/usr/bin/qemu-system-aarch64',
3029     x86_64 => '/usr/bin/qemu-system-x86_64',
3030 };
3031 sub get_command_for_arch($) {
3032     my ($arch) = @_;
3033     return '/usr/bin/kvm' if is_native($arch);
3034
3035     my $cmd = $Arch2Qemu->{$arch}
3036         or die "don't know how to emulate architecture '$arch'\n";
3037     return $cmd;
3038 }
3039
3040 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3041 # to use in a QEMU command line (-cpu element), first array_intersect the result
3042 # of query_supported_ with query_understood_. This is necessary because:
3043 #
3044 # a) query_understood_ returns flags the host cannot use and
3045 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3046 #    flags, but CPU settings - with most of them being flags. Those settings
3047 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3048 #
3049 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3050 # expensive. If you need the value returned from this, you can get it much
3051 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3052 # $accel being 'kvm' or 'tcg'.
3053 #
3054 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3055 # changes, automatically populating pmxcfs.
3056 #
3057 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3058 # since kvm and tcg machines support different flags
3059 #
3060 sub query_supported_cpu_flags {
3061     my ($arch) = @_;
3062
3063     $arch //= get_host_arch();
3064     my $default_machine = $default_machines->{$arch};
3065
3066     my $flags = {};
3067
3068     # FIXME: Once this is merged, the code below should work for ARM as well:
3069     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3070     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3071         $arch eq "aarch64";
3072
3073     my $kvm_supported = defined(kvm_version());
3074     my $qemu_cmd = get_command_for_arch($arch);
3075     my $fakevmid = -1;
3076     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3077
3078     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3079     my $query_supported_run_qemu = sub {
3080         my ($kvm) = @_;
3081
3082         my $flags = {};
3083         my $cmd = [
3084             $qemu_cmd,
3085             '-machine', $default_machine,
3086             '-display', 'none',
3087             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3088             '-mon', 'chardev=qmp,mode=control',
3089             '-pidfile', $pidfile,
3090             '-S', '-daemonize'
3091         ];
3092
3093         if (!$kvm) {
3094             push @$cmd, '-accel', 'tcg';
3095         }
3096
3097         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3098         die "QEMU flag querying VM exited with code " . $rc if $rc;
3099
3100         eval {
3101             my $cmd_result = mon_cmd(
3102                 $fakevmid,
3103                 'query-cpu-model-expansion',
3104                 type => 'full',
3105                 model => { name => 'host' }
3106             );
3107
3108             my $props = $cmd_result->{model}->{props};
3109             foreach my $prop (keys %$props) {
3110                 next if $props->{$prop} ne '1';
3111                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3112                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3113                 # We only keep those with underscores, to match /proc/cpuinfo
3114                 $prop =~ s/\.|-/_/g;
3115                 $flags->{$prop} = 1;
3116             }
3117         };
3118         my $err = $@;
3119
3120         # force stop with 10 sec timeout and 'nocheck'
3121         # always stop, even if QMP failed
3122         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3123
3124         die $err if $err;
3125
3126         return [ sort keys %$flags ];
3127     };
3128
3129     # We need to query QEMU twice, since KVM and TCG have different supported flags
3130     PVE::QemuConfig->lock_config($fakevmid, sub {
3131         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3132         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3133
3134         if ($kvm_supported) {
3135             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3136             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3137         }
3138     });
3139
3140     return $flags;
3141 }
3142
3143 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3144 my $understood_cpu_flag_dir = "/usr/share/kvm";
3145 sub query_understood_cpu_flags {
3146     my $arch = get_host_arch();
3147     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3148
3149     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3150         if ! -e $filepath;
3151
3152     my $raw = file_get_contents($filepath);
3153     $raw =~ s/^\s+|\s+$//g;
3154     my @flags = split(/\s+/, $raw);
3155
3156     return \@flags;
3157 }
3158
3159 sub config_to_command {
3160     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3161         $pbs_backing) = @_;
3162
3163     my $cmd = [];
3164     my $globalFlags = [];
3165     my $machineFlags = [];
3166     my $rtcFlags = [];
3167     my $devices = [];
3168     my $pciaddr = '';
3169     my $bridges = {};
3170     my $ostype = $conf->{ostype};
3171     my $winversion = windows_version($ostype);
3172     my $kvm = $conf->{kvm};
3173     my $nodename = nodename();
3174
3175     my $arch = get_vm_arch($conf);
3176     my $kvm_binary = get_command_for_arch($arch);
3177     my $kvmver = kvm_user_version($kvm_binary);
3178
3179     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3180         $kvmver //= "undefined";
3181         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3182     }
3183
3184     my $add_pve_version = min_version($kvmver, 4, 1);
3185
3186     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3187     my $machine_version = extract_version($machine_type, $kvmver);
3188     $kvm //= 1 if is_native($arch);
3189
3190     $machine_version =~ m/(\d+)\.(\d+)/;
3191     my ($machine_major, $machine_minor) = ($1, $2);
3192
3193     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3194         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3195     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3196         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3197             ." please upgrade node '$nodename'\n"
3198     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3199         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3200         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3201             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3202             ." node '$nodename'\n";
3203     }
3204
3205     # if a specific +pve version is required for a feature, use $version_guard
3206     # instead of min_version to allow machines to be run with the minimum
3207     # required version
3208     my $required_pve_version = 0;
3209     my $version_guard = sub {
3210         my ($major, $minor, $pve) = @_;
3211         return 0 if !min_version($machine_version, $major, $minor, $pve);
3212         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3213         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3214         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3215         return 1;
3216     };
3217
3218     if ($kvm && !defined kvm_version()) {
3219         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3220             ." or enable in BIOS.\n";
3221     }
3222
3223     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3224     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3225     my $use_old_bios_files = undef;
3226     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3227
3228     my $cpuunits = defined($conf->{cpuunits}) ?
3229             $conf->{cpuunits} : $defaults->{cpuunits};
3230
3231     push @$cmd, $kvm_binary;
3232
3233     push @$cmd, '-id', $vmid;
3234
3235     my $vmname = $conf->{name} || "vm$vmid";
3236
3237     push @$cmd, '-name', $vmname;
3238
3239     push @$cmd, '-no-shutdown';
3240
3241     my $use_virtio = 0;
3242
3243     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3244     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3245     push @$cmd, '-mon', "chardev=qmp,mode=control";
3246
3247     if (min_version($machine_version, 2, 12)) {
3248         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3249         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3250     }
3251
3252     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3253
3254     push @$cmd, '-daemonize';
3255
3256     if ($conf->{smbios1}) {
3257         my $smbios_conf = parse_smbios1($conf->{smbios1});
3258         if ($smbios_conf->{base64}) {
3259             # Do not pass base64 flag to qemu
3260             delete $smbios_conf->{base64};
3261             my $smbios_string = "";
3262             foreach my $key (keys %$smbios_conf) {
3263                 my $value;
3264                 if ($key eq "uuid") {
3265                     $value = $smbios_conf->{uuid}
3266                 } else {
3267                     $value = decode_base64($smbios_conf->{$key});
3268                 }
3269                 # qemu accepts any binary data, only commas need escaping by double comma
3270                 $value =~ s/,/,,/g;
3271                 $smbios_string .= "," . $key . "=" . $value if $value;
3272             }
3273             push @$cmd, '-smbios', "type=1" . $smbios_string;
3274         } else {
3275             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3276         }
3277     }
3278
3279     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3280         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3281         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3282
3283         my ($path, $format);
3284         if (my $efidisk = $conf->{efidisk0}) {
3285             my $d = parse_drive('efidisk0', $efidisk);
3286             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3287             $format = $d->{format};
3288             if ($storeid) {
3289                 $path = PVE::Storage::path($storecfg, $d->{file});
3290                 if (!defined($format)) {
3291                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3292                     $format = qemu_img_format($scfg, $volname);
3293                 }
3294             } else {
3295                 $path = $d->{file};
3296                 die "efidisk format must be specified\n"
3297                     if !defined($format);
3298             }
3299         } else {
3300             warn "no efidisk configured! Using temporary efivars disk.\n";
3301             $path = "/tmp/$vmid-ovmf.fd";
3302             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3303             $format = 'raw';
3304         }
3305
3306         my $size_str = "";
3307
3308         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3309             $size_str = ",size=" . (-s $ovmf_vars);
3310         }
3311
3312         # on slower ceph clusters, booting without cache on efidisk can take a while, see #3329
3313         my $cache = $path =~ m/^rbd:/ ? ',cache=writeback' : '';
3314
3315         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3316         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=$path";
3317     }
3318
3319     # load q35 config
3320     if ($q35) {
3321         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3322         if (min_version($machine_version, 4, 0)) {
3323             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3324         } else {
3325             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3326         }
3327     }
3328
3329     if ($conf->{vmgenid}) {
3330         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3331     }
3332
3333     # add usb controllers
3334     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3335         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3336     push @$devices, @usbcontrollers if @usbcontrollers;
3337     my $vga = parse_vga($conf->{vga});
3338
3339     my $qxlnum = vga_conf_has_spice($conf->{vga});
3340     $vga->{type} = 'qxl' if $qxlnum;
3341
3342     if (!$vga->{type}) {
3343         if ($arch eq 'aarch64') {
3344             $vga->{type} = 'virtio';
3345         } elsif (min_version($machine_version, 2, 9)) {
3346             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3347         } else {
3348             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3349         }
3350     }
3351
3352     # enable absolute mouse coordinates (needed by vnc)
3353     my $tablet;
3354     if (defined($conf->{tablet})) {
3355         $tablet = $conf->{tablet};
3356     } else {
3357         $tablet = $defaults->{tablet};
3358         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3359         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3360     }
3361
3362     if ($tablet) {
3363         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3364         my $kbd = print_keyboarddevice_full($conf, $arch);
3365         push @$devices, '-device', $kbd if defined($kbd);
3366     }
3367
3368     my $bootorder = device_bootorder($conf);
3369
3370     # host pci device passthrough
3371     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3372         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3373
3374     # usb devices
3375     my $usb_dev_features = {};
3376     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3377
3378     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3379         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3380     push @$devices, @usbdevices if @usbdevices;
3381
3382     # serial devices
3383     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3384         if (my $path = $conf->{"serial$i"}) {
3385             if ($path eq 'socket') {
3386                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3387                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3388                 # On aarch64, serial0 is the UART device. Qemu only allows
3389                 # connecting UART devices via the '-serial' command line, as
3390                 # the device has a fixed slot on the hardware...
3391                 if ($arch eq 'aarch64' && $i == 0) {
3392                     push @$devices, '-serial', "chardev:serial$i";
3393                 } else {
3394                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3395                 }
3396             } else {
3397                 die "no such serial device\n" if ! -c $path;
3398                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3399                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3400             }
3401         }
3402     }
3403
3404     # parallel devices
3405     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3406         if (my $path = $conf->{"parallel$i"}) {
3407             die "no such parallel device\n" if ! -c $path;
3408             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3409             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3410             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3411         }
3412     }
3413
3414     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3415         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3416         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3417         push @$devices, @$audio_devs;
3418     }
3419
3420     my $sockets = 1;
3421     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3422     $sockets = $conf->{sockets} if  $conf->{sockets};
3423
3424     my $cores = $conf->{cores} || 1;
3425
3426     my $maxcpus = $sockets * $cores;
3427
3428     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3429
3430     my $allowed_vcpus = $cpuinfo->{cpus};
3431
3432     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3433         if ($allowed_vcpus < $maxcpus);
3434
3435     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3436
3437         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3438         for (my $i = 2; $i <= $vcpus; $i++)  {
3439             my $cpustr = print_cpu_device($conf,$i);
3440             push @$cmd, '-device', $cpustr;
3441         }
3442
3443     } else {
3444
3445         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3446     }
3447     push @$cmd, '-nodefaults';
3448
3449     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3450
3451     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3452
3453     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3454
3455     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3456         push @$devices, '-device', print_vga_device(
3457             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3458         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3459         push @$cmd,  '-vnc', "unix:$socket,password=on";
3460     } else {
3461         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3462         push @$cmd, '-nographic';
3463     }
3464
3465     # time drift fix
3466     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3467     my $useLocaltime = $conf->{localtime};
3468
3469     if ($winversion >= 5) { # windows
3470         $useLocaltime = 1 if !defined($conf->{localtime});
3471
3472         # use time drift fix when acpi is enabled
3473         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3474             $tdf = 1 if !defined($conf->{tdf});
3475         }
3476     }
3477
3478     if ($winversion >= 6) {
3479         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3480         push @$cmd, '-no-hpet';
3481     }
3482
3483     push @$rtcFlags, 'driftfix=slew' if $tdf;
3484
3485     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3486         push @$rtcFlags, "base=$conf->{startdate}";
3487     } elsif ($useLocaltime) {
3488         push @$rtcFlags, 'base=localtime';
3489     }
3490
3491     if ($forcecpu) {
3492         push @$cmd, '-cpu', $forcecpu;
3493     } else {
3494         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3495     }
3496
3497     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3498
3499     push @$cmd, '-S' if $conf->{freeze};
3500
3501     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3502
3503     my $guest_agent = parse_guest_agent($conf);
3504
3505     if ($guest_agent->{enabled}) {
3506         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3507         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3508
3509         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3510             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3511             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3512             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3513         } elsif ($guest_agent->{type} eq 'isa') {
3514             push @$devices, '-device', "isa-serial,chardev=qga0";
3515         }
3516     }
3517
3518     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3519     if ($rng && $version_guard->(4, 1, 2)) {
3520         check_rng_source($rng->{source});
3521
3522         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3523         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3524         my $limiter_str = "";
3525         if ($max_bytes) {
3526             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3527         }
3528
3529         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3530         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3531         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3532     }
3533
3534     my $spice_port;
3535
3536     if ($qxlnum) {
3537         if ($qxlnum > 1) {
3538             if ($winversion){
3539                 for (my $i = 1; $i < $qxlnum; $i++){
3540                     push @$devices, '-device', print_vga_device(
3541                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3542                 }
3543             } else {
3544                 # assume other OS works like Linux
3545                 my ($ram, $vram) = ("134217728", "67108864");
3546                 if ($vga->{memory}) {
3547                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3548                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3549                 }
3550                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3551                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3552             }
3553         }
3554
3555         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3556
3557         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3558         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3559         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3560
3561         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3562         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3563         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3564
3565         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3566         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3567
3568         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3569         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3570         if ($spice_enhancement->{foldersharing}) {
3571             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3572             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3573         }
3574
3575         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3576         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3577             if $spice_enhancement->{videostreaming};
3578
3579         push @$devices, '-spice', "$spice_opts";
3580     }
3581
3582     # enable balloon by default, unless explicitly disabled
3583     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3584         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3585         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3586     }
3587
3588     if ($conf->{watchdog}) {
3589         my $wdopts = parse_watchdog($conf->{watchdog});
3590         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3591         my $watchdog = $wdopts->{model} || 'i6300esb';
3592         push @$devices, '-device', "$watchdog$pciaddr";
3593         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3594     }
3595
3596     my $vollist = [];
3597     my $scsicontroller = {};
3598     my $ahcicontroller = {};
3599     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3600
3601     # Add iscsi initiator name if available
3602     if (my $initiator = get_initiator_name()) {
3603         push @$devices, '-iscsi', "initiator-name=$initiator";
3604     }
3605
3606     PVE::QemuConfig->foreach_volume($conf, sub {
3607         my ($ds, $drive) = @_;
3608
3609         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3610             push @$vollist, $drive->{file};
3611         }
3612
3613         # ignore efidisk here, already added in bios/fw handling code above
3614         return if $drive->{interface} eq 'efidisk';
3615
3616         $use_virtio = 1 if $ds =~ m/^virtio/;
3617
3618         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3619
3620         if ($drive->{interface} eq 'virtio'){
3621            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3622         }
3623
3624         if ($drive->{interface} eq 'scsi') {
3625
3626             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3627
3628             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3629                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3630
3631             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3632             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3633
3634             my $iothread = '';
3635             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3636                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3637                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3638             } elsif ($drive->{iothread}) {
3639                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3640             }
3641
3642             my $queues = '';
3643             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3644                 $queues = ",num_queues=$drive->{queues}";
3645             }
3646
3647             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3648                 if !$scsicontroller->{$controller};
3649             $scsicontroller->{$controller}=1;
3650         }
3651
3652         if ($drive->{interface} eq 'sata') {
3653             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3654             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3655             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3656                 if !$ahcicontroller->{$controller};
3657             $ahcicontroller->{$controller}=1;
3658         }
3659
3660         my $pbs_conf = $pbs_backing->{$ds};
3661         my $pbs_name = undef;
3662         if ($pbs_conf) {
3663             $pbs_name = "drive-$ds-pbs";
3664             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3665         }
3666
3667         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive, $pbs_name);
3668
3669         # extra protection for templates, but SATA and IDE don't support it..
3670         my $read_only = PVE::QemuConfig->is_template($conf)
3671             && $drive->{interface} ne 'sata'
3672             && $drive->{interface} ne 'ide';
3673
3674         $drive_cmd .= ',readonly=on' if $read_only;
3675
3676         push @$devices, '-drive',$drive_cmd;
3677         push @$devices, '-device', print_drivedevice_full(
3678             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3679     });
3680
3681     for (my $i = 0; $i < $MAX_NETS; $i++) {
3682         my $netname = "net$i";
3683
3684         next if !$conf->{$netname};
3685         my $d = parse_net($conf->{$netname});
3686         next if !$d;
3687
3688         $use_virtio = 1 if $d->{model} eq 'virtio';
3689
3690         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3691
3692         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3693         push @$devices, '-netdev', $netdevfull;
3694
3695         my $netdevicefull = print_netdevice_full(
3696             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3697
3698         push @$devices, '-device', $netdevicefull;
3699     }
3700
3701     if ($conf->{ivshmem}) {
3702         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3703
3704         my $bus;
3705         if ($q35) {
3706             $bus = print_pcie_addr("ivshmem");
3707         } else {
3708             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3709         }
3710
3711         my $ivshmem_name = $ivshmem->{name} // $vmid;
3712         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3713
3714         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3715         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3716             .",size=$ivshmem->{size}M";
3717     }
3718
3719     # pci.4 is nested in pci.1
3720     $bridges->{1} = 1 if $bridges->{4};
3721
3722     if (!$q35) {
3723         # add pci bridges
3724         if (min_version($machine_version, 2, 3)) {
3725            $bridges->{1} = 1;
3726            $bridges->{2} = 1;
3727         }
3728
3729         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3730
3731     }
3732
3733     for my $k (sort {$b cmp $a} keys %$bridges) {
3734         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3735
3736         my $k_name = $k;
3737         if ($k == 2 && $legacy_igd) {
3738             $k_name = "$k-igd";
3739         }
3740         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3741
3742         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3743         if ($q35) {
3744             # add after -readconfig pve-q35.cfg
3745             splice @$devices, 2, 0, '-device', $devstr;
3746         } else {
3747             unshift @$devices, '-device', $devstr if $k > 0;
3748         }
3749     }
3750
3751     if (!$kvm) {
3752         push @$machineFlags, 'accel=tcg';
3753     }
3754
3755     my $machine_type_min = $machine_type;
3756     if ($add_pve_version) {
3757         $machine_type_min =~ s/\+pve\d+$//;
3758         $machine_type_min .= "+pve$required_pve_version";
3759     }
3760     push @$machineFlags, "type=${machine_type_min}";
3761
3762     push @$cmd, @$devices;
3763     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3764     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3765     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3766
3767     if (my $vmstate = $conf->{vmstate}) {
3768         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3769         push @$vollist, $vmstate;
3770         push @$cmd, '-loadstate', $statepath;
3771         print "activating and using '$vmstate' as vmstate\n";
3772     }
3773
3774     # add custom args
3775     if ($conf->{args}) {
3776         my $aa = PVE::Tools::split_args($conf->{args});
3777         push @$cmd, @$aa;
3778     }
3779
3780     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3781 }
3782
3783 sub check_rng_source {
3784     my ($source) = @_;
3785
3786     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3787     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3788         if ! -e $source;
3789
3790     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3791     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3792         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3793         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3794         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3795             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3796             ." to the host.\n";
3797     }
3798 }
3799
3800 sub spice_port {
3801     my ($vmid) = @_;
3802
3803     my $res = mon_cmd($vmid, 'query-spice');
3804
3805     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3806 }
3807
3808 sub vm_devices_list {
3809     my ($vmid) = @_;
3810
3811     my $res = mon_cmd($vmid, 'query-pci');
3812     my $devices_to_check = [];
3813     my $devices = {};
3814     foreach my $pcibus (@$res) {
3815         push @$devices_to_check, @{$pcibus->{devices}},
3816     }
3817
3818     while (@$devices_to_check) {
3819         my $to_check = [];
3820         for my $d (@$devices_to_check) {
3821             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3822             next if !$d->{'pci_bridge'};
3823
3824             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3825             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3826         }
3827         $devices_to_check = $to_check;
3828     }
3829
3830     my $resblock = mon_cmd($vmid, 'query-block');
3831     foreach my $block (@$resblock) {
3832         if($block->{device} =~ m/^drive-(\S+)/){
3833                 $devices->{$1} = 1;
3834         }
3835     }
3836
3837     my $resmice = mon_cmd($vmid, 'query-mice');
3838     foreach my $mice (@$resmice) {
3839         if ($mice->{name} eq 'QEMU HID Tablet') {
3840             $devices->{tablet} = 1;
3841             last;
3842         }
3843     }
3844
3845     # for usb devices there is no query-usb
3846     # but we can iterate over the entries in
3847     # qom-list path=/machine/peripheral
3848     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3849     foreach my $per (@$resperipheral) {
3850         if ($per->{name} =~ m/^usb\d+$/) {
3851             $devices->{$per->{name}} = 1;
3852         }
3853     }
3854
3855     return $devices;
3856 }
3857
3858 sub vm_deviceplug {
3859     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3860
3861     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3862
3863     my $devices_list = vm_devices_list($vmid);
3864     return 1 if defined($devices_list->{$deviceid});
3865
3866     # add PCI bridge if we need it for the device
3867     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3868
3869     if ($deviceid eq 'tablet') {
3870
3871         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3872
3873     } elsif ($deviceid eq 'keyboard') {
3874
3875         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3876
3877     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3878
3879         die "usb hotplug currently not reliable\n";
3880         # since we can't reliably hot unplug all added usb devices and usb
3881         # passthrough breaks live migration we disable usb hotplugging for now
3882         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3883
3884     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3885
3886         qemu_iothread_add($vmid, $deviceid, $device);
3887
3888         qemu_driveadd($storecfg, $vmid, $device);
3889         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3890
3891         qemu_deviceadd($vmid, $devicefull);
3892         eval { qemu_deviceaddverify($vmid, $deviceid); };
3893         if (my $err = $@) {
3894             eval { qemu_drivedel($vmid, $deviceid); };
3895             warn $@ if $@;
3896             die $err;
3897         }
3898
3899     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3900
3901
3902         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3903         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3904         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3905
3906         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3907
3908         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3909             qemu_iothread_add($vmid, $deviceid, $device);
3910             $devicefull .= ",iothread=iothread-$deviceid";
3911         }
3912
3913         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3914             $devicefull .= ",num_queues=$device->{queues}";
3915         }
3916
3917         qemu_deviceadd($vmid, $devicefull);
3918         qemu_deviceaddverify($vmid, $deviceid);
3919
3920     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3921
3922         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3923         qemu_driveadd($storecfg, $vmid, $device);
3924
3925         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3926         eval { qemu_deviceadd($vmid, $devicefull); };
3927         if (my $err = $@) {
3928             eval { qemu_drivedel($vmid, $deviceid); };
3929             warn $@ if $@;
3930             die $err;
3931         }
3932
3933     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3934
3935         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3936
3937         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3938         my $use_old_bios_files = undef;
3939         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3940
3941         my $netdevicefull = print_netdevice_full(
3942             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3943         qemu_deviceadd($vmid, $netdevicefull);
3944         eval {
3945             qemu_deviceaddverify($vmid, $deviceid);
3946             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3947         };
3948         if (my $err = $@) {
3949             eval { qemu_netdevdel($vmid, $deviceid); };
3950             warn $@ if $@;
3951             die $err;
3952         }
3953
3954     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3955
3956         my $bridgeid = $2;
3957         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3958         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3959
3960         qemu_deviceadd($vmid, $devicefull);
3961         qemu_deviceaddverify($vmid, $deviceid);
3962
3963     } else {
3964         die "can't hotplug device '$deviceid'\n";
3965     }
3966
3967     return 1;
3968 }
3969
3970 # fixme: this should raise exceptions on error!
3971 sub vm_deviceunplug {
3972     my ($vmid, $conf, $deviceid) = @_;
3973
3974     my $devices_list = vm_devices_list($vmid);
3975     return 1 if !defined($devices_list->{$deviceid});
3976
3977     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3978     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3979
3980     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3981
3982         qemu_devicedel($vmid, $deviceid);
3983
3984     } elsif ($deviceid =~ m/^usb\d+$/) {
3985
3986         die "usb hotplug currently not reliable\n";
3987         # when unplugging usb devices this way, there may be remaining usb
3988         # controllers/hubs so we disable it for now
3989         #qemu_devicedel($vmid, $deviceid);
3990         #qemu_devicedelverify($vmid, $deviceid);
3991
3992     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3993
3994         qemu_devicedel($vmid, $deviceid);
3995         qemu_devicedelverify($vmid, $deviceid);
3996         qemu_drivedel($vmid, $deviceid);
3997         qemu_iothread_del($conf, $vmid, $deviceid);
3998
3999     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4000
4001         qemu_devicedel($vmid, $deviceid);
4002         qemu_devicedelverify($vmid, $deviceid);
4003         qemu_iothread_del($conf, $vmid, $deviceid);
4004
4005     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4006
4007         qemu_devicedel($vmid, $deviceid);
4008         qemu_drivedel($vmid, $deviceid);
4009         qemu_deletescsihw($conf, $vmid, $deviceid);
4010
4011     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4012
4013         qemu_devicedel($vmid, $deviceid);
4014         qemu_devicedelverify($vmid, $deviceid);
4015         qemu_netdevdel($vmid, $deviceid);
4016
4017     } else {
4018         die "can't unplug device '$deviceid'\n";
4019     }
4020
4021     return 1;
4022 }
4023
4024 sub qemu_deviceadd {
4025     my ($vmid, $devicefull) = @_;
4026
4027     $devicefull = "driver=".$devicefull;
4028     my %options =  split(/[=,]/, $devicefull);
4029
4030     mon_cmd($vmid, "device_add" , %options);
4031 }
4032
4033 sub qemu_devicedel {
4034     my ($vmid, $deviceid) = @_;
4035
4036     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4037 }
4038
4039 sub qemu_iothread_add {
4040     my($vmid, $deviceid, $device) = @_;
4041
4042     if ($device->{iothread}) {
4043         my $iothreads = vm_iothreads_list($vmid);
4044         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4045     }
4046 }
4047
4048 sub qemu_iothread_del {
4049     my($conf, $vmid, $deviceid) = @_;
4050
4051     my $confid = $deviceid;
4052     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4053         $confid = 'scsi' . $1;
4054     }
4055     my $device = parse_drive($confid, $conf->{$confid});
4056     if ($device->{iothread}) {
4057         my $iothreads = vm_iothreads_list($vmid);
4058         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4059     }
4060 }
4061
4062 sub qemu_objectadd {
4063     my($vmid, $objectid, $qomtype) = @_;
4064
4065     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4066
4067     return 1;
4068 }
4069
4070 sub qemu_objectdel {
4071     my($vmid, $objectid) = @_;
4072
4073     mon_cmd($vmid, "object-del", id => $objectid);
4074
4075     return 1;
4076 }
4077
4078 sub qemu_driveadd {
4079     my ($storecfg, $vmid, $device) = @_;
4080
4081     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4082     $drive =~ s/\\/\\\\/g;
4083     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4084
4085     # If the command succeeds qemu prints: "OK"
4086     return 1 if $ret =~ m/OK/s;
4087
4088     die "adding drive failed: $ret\n";
4089 }
4090
4091 sub qemu_drivedel {
4092     my($vmid, $deviceid) = @_;
4093
4094     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4095     $ret =~ s/^\s+//;
4096
4097     return 1 if $ret eq "";
4098
4099     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4100     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4101
4102     die "deleting drive $deviceid failed : $ret\n";
4103 }
4104
4105 sub qemu_deviceaddverify {
4106     my ($vmid, $deviceid) = @_;
4107
4108     for (my $i = 0; $i <= 5; $i++) {
4109          my $devices_list = vm_devices_list($vmid);
4110          return 1 if defined($devices_list->{$deviceid});
4111          sleep 1;
4112     }
4113
4114     die "error on hotplug device '$deviceid'\n";
4115 }
4116
4117
4118 sub qemu_devicedelverify {
4119     my ($vmid, $deviceid) = @_;
4120
4121     # need to verify that the device is correctly removed as device_del
4122     # is async and empty return is not reliable
4123
4124     for (my $i = 0; $i <= 5; $i++) {
4125          my $devices_list = vm_devices_list($vmid);
4126          return 1 if !defined($devices_list->{$deviceid});
4127          sleep 1;
4128     }
4129
4130     die "error on hot-unplugging device '$deviceid'\n";
4131 }
4132
4133 sub qemu_findorcreatescsihw {
4134     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4135
4136     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4137
4138     my $scsihwid="$controller_prefix$controller";
4139     my $devices_list = vm_devices_list($vmid);
4140
4141     if(!defined($devices_list->{$scsihwid})) {
4142         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4143     }
4144
4145     return 1;
4146 }
4147
4148 sub qemu_deletescsihw {
4149     my ($conf, $vmid, $opt) = @_;
4150
4151     my $device = parse_drive($opt, $conf->{$opt});
4152
4153     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4154         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4155         return 1;
4156     }
4157
4158     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4159
4160     my $devices_list = vm_devices_list($vmid);
4161     foreach my $opt (keys %{$devices_list}) {
4162         if (is_valid_drivename($opt)) {
4163             my $drive = parse_drive($opt, $conf->{$opt});
4164             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4165                 return 1;
4166             }
4167         }
4168     }
4169
4170     my $scsihwid="scsihw$controller";
4171
4172     vm_deviceunplug($vmid, $conf, $scsihwid);
4173
4174     return 1;
4175 }
4176
4177 sub qemu_add_pci_bridge {
4178     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4179
4180     my $bridges = {};
4181
4182     my $bridgeid;
4183
4184     print_pci_addr($device, $bridges, $arch, $machine_type);
4185
4186     while (my ($k, $v) = each %$bridges) {
4187         $bridgeid = $k;
4188     }
4189     return 1 if !defined($bridgeid) || $bridgeid < 1;
4190
4191     my $bridge = "pci.$bridgeid";
4192     my $devices_list = vm_devices_list($vmid);
4193
4194     if (!defined($devices_list->{$bridge})) {
4195         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4196     }
4197
4198     return 1;
4199 }
4200
4201 sub qemu_set_link_status {
4202     my ($vmid, $device, $up) = @_;
4203
4204     mon_cmd($vmid, "set_link", name => $device,
4205                up => $up ? JSON::true : JSON::false);
4206 }
4207
4208 sub qemu_netdevadd {
4209     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4210
4211     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4212     my %options =  split(/[=,]/, $netdev);
4213
4214     if (defined(my $vhost = $options{vhost})) {
4215         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4216     }
4217
4218     if (defined(my $queues = $options{queues})) {
4219         $options{queues} = $queues + 0;
4220     }
4221
4222     mon_cmd($vmid, "netdev_add",  %options);
4223     return 1;
4224 }
4225
4226 sub qemu_netdevdel {
4227     my ($vmid, $deviceid) = @_;
4228
4229     mon_cmd($vmid, "netdev_del", id => $deviceid);
4230 }
4231
4232 sub qemu_usb_hotplug {
4233     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4234
4235     return if !$device;
4236
4237     # remove the old one first
4238     vm_deviceunplug($vmid, $conf, $deviceid);
4239
4240     # check if xhci controller is necessary and available
4241     if ($device->{usb3}) {
4242
4243         my $devicelist = vm_devices_list($vmid);
4244
4245         if (!$devicelist->{xhci}) {
4246             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4247             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4248         }
4249     }
4250     my $d = parse_usb_device($device->{host});
4251     $d->{usb3} = $device->{usb3};
4252
4253     # add the new one
4254     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4255 }
4256
4257 sub qemu_cpu_hotplug {
4258     my ($vmid, $conf, $vcpus) = @_;
4259
4260     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4261
4262     my $sockets = 1;
4263     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4264     $sockets = $conf->{sockets} if  $conf->{sockets};
4265     my $cores = $conf->{cores} || 1;
4266     my $maxcpus = $sockets * $cores;
4267
4268     $vcpus = $maxcpus if !$vcpus;
4269
4270     die "you can't add more vcpus than maxcpus\n"
4271         if $vcpus > $maxcpus;
4272
4273     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4274
4275     if ($vcpus < $currentvcpus) {
4276
4277         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4278
4279             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4280                 qemu_devicedel($vmid, "cpu$i");
4281                 my $retry = 0;
4282                 my $currentrunningvcpus = undef;
4283                 while (1) {
4284                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4285                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4286                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4287                     $retry++;
4288                     sleep 1;
4289                 }
4290                 #update conf after each succesfull cpu unplug
4291                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4292                 PVE::QemuConfig->write_config($vmid, $conf);
4293             }
4294         } else {
4295             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4296         }
4297
4298         return;
4299     }
4300
4301     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4302     die "vcpus in running vm does not match its configuration\n"
4303         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4304
4305     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4306
4307         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4308             my $cpustr = print_cpu_device($conf, $i);
4309             qemu_deviceadd($vmid, $cpustr);
4310
4311             my $retry = 0;
4312             my $currentrunningvcpus = undef;
4313             while (1) {
4314                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4315                 last if scalar(@{$currentrunningvcpus}) == $i;
4316                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4317                 sleep 1;
4318                 $retry++;
4319             }
4320             #update conf after each succesfull cpu hotplug
4321             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4322             PVE::QemuConfig->write_config($vmid, $conf);
4323         }
4324     } else {
4325
4326         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4327             mon_cmd($vmid, "cpu-add", id => int($i));
4328         }
4329     }
4330 }
4331
4332 sub qemu_block_set_io_throttle {
4333     my ($vmid, $deviceid,
4334         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4335         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4336         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4337         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4338
4339     return if !check_running($vmid) ;
4340
4341     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4342         bps => int($bps),
4343         bps_rd => int($bps_rd),
4344         bps_wr => int($bps_wr),
4345         iops => int($iops),
4346         iops_rd => int($iops_rd),
4347         iops_wr => int($iops_wr),
4348         bps_max => int($bps_max),
4349         bps_rd_max => int($bps_rd_max),
4350         bps_wr_max => int($bps_wr_max),
4351         iops_max => int($iops_max),
4352         iops_rd_max => int($iops_rd_max),
4353         iops_wr_max => int($iops_wr_max),
4354         bps_max_length => int($bps_max_length),
4355         bps_rd_max_length => int($bps_rd_max_length),
4356         bps_wr_max_length => int($bps_wr_max_length),
4357         iops_max_length => int($iops_max_length),
4358         iops_rd_max_length => int($iops_rd_max_length),
4359         iops_wr_max_length => int($iops_wr_max_length),
4360     );
4361
4362 }
4363
4364 sub qemu_block_resize {
4365     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4366
4367     my $running = check_running($vmid);
4368
4369     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4370
4371     return if !$running;
4372
4373     my $padding = (1024 - $size % 1024) % 1024;
4374     $size = $size + $padding;
4375
4376     mon_cmd(
4377         $vmid,
4378         "block_resize",
4379         device => $deviceid,
4380         size => int($size),
4381         timeout => 60,
4382     );
4383 }
4384
4385 sub qemu_volume_snapshot {
4386     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4387
4388     my $running = check_running($vmid);
4389
4390     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4391         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4392     } else {
4393         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4394     }
4395 }
4396
4397 sub qemu_volume_snapshot_delete {
4398     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4399
4400     my $running = check_running($vmid);
4401
4402     if($running) {
4403
4404         $running = undef;
4405         my $conf = PVE::QemuConfig->load_config($vmid);
4406         PVE::QemuConfig->foreach_volume($conf, sub {
4407             my ($ds, $drive) = @_;
4408             $running = 1 if $drive->{file} eq $volid;
4409         });
4410     }
4411
4412     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4413         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4414     } else {
4415         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4416     }
4417 }
4418
4419 sub set_migration_caps {
4420     my ($vmid, $savevm) = @_;
4421
4422     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4423
4424     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4425     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4426
4427     my $cap_ref = [];
4428
4429     my $enabled_cap = {
4430         "auto-converge" => 1,
4431         "xbzrle" => 1,
4432         "x-rdma-pin-all" => 0,
4433         "zero-blocks" => 0,
4434         "compress" => 0,
4435         "dirty-bitmaps" => $dirty_bitmaps,
4436     };
4437
4438     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4439
4440     for my $supported_capability (@$supported_capabilities) {
4441         push @$cap_ref, {
4442             capability => $supported_capability->{capability},
4443             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4444         };
4445     }
4446
4447     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4448 }
4449
4450 sub foreach_volid {
4451     my ($conf, $func, @param) = @_;
4452
4453     my $volhash = {};
4454
4455     my $test_volid = sub {
4456         my ($key, $drive, $snapname) = @_;
4457
4458         my $volid = $drive->{file};
4459         return if !$volid;
4460
4461         $volhash->{$volid}->{cdrom} //= 1;
4462         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4463
4464         my $replicate = $drive->{replicate} // 1;
4465         $volhash->{$volid}->{replicate} //= 0;
4466         $volhash->{$volid}->{replicate} = 1 if $replicate;
4467
4468         $volhash->{$volid}->{shared} //= 0;
4469         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4470
4471         $volhash->{$volid}->{referenced_in_config} //= 0;
4472         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4473
4474         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4475             if defined($snapname);
4476
4477         my $size = $drive->{size};
4478         $volhash->{$volid}->{size} //= $size if $size;
4479
4480         $volhash->{$volid}->{is_vmstate} //= 0;
4481         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4482
4483         $volhash->{$volid}->{is_unused} //= 0;
4484         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4485
4486         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4487     };
4488
4489     my $include_opts = {
4490         extra_keys => ['vmstate'],
4491         include_unused => 1,
4492     };
4493
4494     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4495     foreach my $snapname (keys %{$conf->{snapshots}}) {
4496         my $snap = $conf->{snapshots}->{$snapname};
4497         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4498     }
4499
4500     foreach my $volid (keys %$volhash) {
4501         &$func($volid, $volhash->{$volid}, @param);
4502     }
4503 }
4504
4505 my $fast_plug_option = {
4506     'lock' => 1,
4507     'name' => 1,
4508     'onboot' => 1,
4509     'shares' => 1,
4510     'startup' => 1,
4511     'description' => 1,
4512     'protection' => 1,
4513     'vmstatestorage' => 1,
4514     'hookscript' => 1,
4515     'tags' => 1,
4516 };
4517
4518 # hotplug changes in [PENDING]
4519 # $selection hash can be used to only apply specified options, for
4520 # example: { cores => 1 } (only apply changed 'cores')
4521 # $errors ref is used to return error messages
4522 sub vmconfig_hotplug_pending {
4523     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4524
4525     my $defaults = load_defaults();
4526     my $arch = get_vm_arch($conf);
4527     my $machine_type = get_vm_machine($conf, undef, $arch);
4528
4529     # commit values which do not have any impact on running VM first
4530     # Note: those option cannot raise errors, we we do not care about
4531     # $selection and always apply them.
4532
4533     my $add_error = sub {
4534         my ($opt, $msg) = @_;
4535         $errors->{$opt} = "hotplug problem - $msg";
4536     };
4537
4538     my $changes = 0;
4539     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4540         if ($fast_plug_option->{$opt}) {
4541             $conf->{$opt} = $conf->{pending}->{$opt};
4542             delete $conf->{pending}->{$opt};
4543             $changes = 1;
4544         }
4545     }
4546
4547     if ($changes) {
4548         PVE::QemuConfig->write_config($vmid, $conf);
4549     }
4550
4551     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4552
4553     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4554     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4555     foreach my $opt (sort keys %$pending_delete_hash) {
4556         next if $selection && !$selection->{$opt};
4557         my $force = $pending_delete_hash->{$opt}->{force};
4558         eval {
4559             if ($opt eq 'hotplug') {
4560                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4561             } elsif ($opt eq 'tablet') {
4562                 die "skip\n" if !$hotplug_features->{usb};
4563                 if ($defaults->{tablet}) {
4564                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4565                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4566                         if $arch eq 'aarch64';
4567                 } else {
4568                     vm_deviceunplug($vmid, $conf, 'tablet');
4569                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4570                 }
4571             } elsif ($opt =~ m/^usb\d+/) {
4572                 die "skip\n";
4573                 # since we cannot reliably hot unplug usb devices we are disabling it
4574                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4575                 #vm_deviceunplug($vmid, $conf, $opt);
4576             } elsif ($opt eq 'vcpus') {
4577                 die "skip\n" if !$hotplug_features->{cpu};
4578                 qemu_cpu_hotplug($vmid, $conf, undef);
4579             } elsif ($opt eq 'balloon') {
4580                 # enable balloon device is not hotpluggable
4581                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4582                 # here we reset the ballooning value to memory
4583                 my $balloon = $conf->{memory} || $defaults->{memory};
4584                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4585             } elsif ($fast_plug_option->{$opt}) {
4586                 # do nothing
4587             } elsif ($opt =~ m/^net(\d+)$/) {
4588                 die "skip\n" if !$hotplug_features->{network};
4589                 vm_deviceunplug($vmid, $conf, $opt);
4590             } elsif (is_valid_drivename($opt)) {
4591                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4592                 vm_deviceunplug($vmid, $conf, $opt);
4593                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4594             } elsif ($opt =~ m/^memory$/) {
4595                 die "skip\n" if !$hotplug_features->{memory};
4596                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4597             } elsif ($opt eq 'cpuunits') {
4598                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4599             } elsif ($opt eq 'cpulimit') {
4600                 $cgroup->change_cpu_quota(-1, 100000);
4601             } else {
4602                 die "skip\n";
4603             }
4604         };
4605         if (my $err = $@) {
4606             &$add_error($opt, $err) if $err ne "skip\n";
4607         } else {
4608             delete $conf->{$opt};
4609             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4610         }
4611     }
4612
4613     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4614     $apply_pending_cloudinit = sub {
4615         return if $apply_pending_cloudinit_done; # once is enough
4616         $apply_pending_cloudinit_done = 1; # once is enough
4617
4618         my ($key, $value) = @_;
4619
4620         my @cloudinit_opts = keys %$confdesc_cloudinit;
4621         foreach my $opt (keys %{$conf->{pending}}) {
4622             next if !grep { $_ eq $opt } @cloudinit_opts;
4623             $conf->{$opt} = delete $conf->{pending}->{$opt};
4624         }
4625
4626         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4627         foreach my $opt (sort keys %$pending_delete_hash) {
4628             next if !grep { $_ eq $opt } @cloudinit_opts;
4629             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4630             delete $conf->{$opt};
4631         }
4632
4633         my $new_conf = { %$conf };
4634         $new_conf->{$key} = $value;
4635         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4636     };
4637
4638     foreach my $opt (keys %{$conf->{pending}}) {
4639         next if $selection && !$selection->{$opt};
4640         my $value = $conf->{pending}->{$opt};
4641         eval {
4642             if ($opt eq 'hotplug') {
4643                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4644             } elsif ($opt eq 'tablet') {
4645                 die "skip\n" if !$hotplug_features->{usb};
4646                 if ($value == 1) {
4647                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4648                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4649                         if $arch eq 'aarch64';
4650                 } elsif ($value == 0) {
4651                     vm_deviceunplug($vmid, $conf, 'tablet');
4652                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4653                 }
4654             } elsif ($opt =~ m/^usb\d+$/) {
4655                 die "skip\n";
4656                 # since we cannot reliably hot unplug usb devices we disable it for now
4657                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4658                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4659                 #die "skip\n" if !$d;
4660                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4661             } elsif ($opt eq 'vcpus') {
4662                 die "skip\n" if !$hotplug_features->{cpu};
4663                 qemu_cpu_hotplug($vmid, $conf, $value);
4664             } elsif ($opt eq 'balloon') {
4665                 # enable/disable balloning device is not hotpluggable
4666                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4667                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4668                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4669
4670                 # allow manual ballooning if shares is set to zero
4671                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4672                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4673                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4674                 }
4675             } elsif ($opt =~ m/^net(\d+)$/) {
4676                 # some changes can be done without hotplug
4677                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4678                                     $vmid, $opt, $value, $arch, $machine_type);
4679             } elsif (is_valid_drivename($opt)) {
4680                 die "skip\n" if $opt eq 'efidisk0';
4681                 # some changes can be done without hotplug
4682                 my $drive = parse_drive($opt, $value);
4683                 if (drive_is_cloudinit($drive)) {
4684                     &$apply_pending_cloudinit($opt, $value);
4685                 }
4686                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4687                                      $vmid, $opt, $value, $arch, $machine_type);
4688             } elsif ($opt =~ m/^memory$/) { #dimms
4689                 die "skip\n" if !$hotplug_features->{memory};
4690                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4691             } elsif ($opt eq 'cpuunits') {
4692                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4693             } elsif ($opt eq 'cpulimit') {
4694                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4695                 $cgroup->change_cpu_quota($cpulimit, 100000);
4696             } else {
4697                 die "skip\n";  # skip non-hot-pluggable options
4698             }
4699         };
4700         if (my $err = $@) {
4701             &$add_error($opt, $err) if $err ne "skip\n";
4702         } else {
4703             $conf->{$opt} = $value;
4704             delete $conf->{pending}->{$opt};
4705         }
4706     }
4707
4708     PVE::QemuConfig->write_config($vmid, $conf);
4709 }
4710
4711 sub try_deallocate_drive {
4712     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4713
4714     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4715         my $volid = $drive->{file};
4716         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4717             my $sid = PVE::Storage::parse_volume_id($volid);
4718             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4719
4720             # check if the disk is really unused
4721             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4722                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4723             PVE::Storage::vdisk_free($storecfg, $volid);
4724             return 1;
4725         } else {
4726             # If vm is not owner of this disk remove from config
4727             return 1;
4728         }
4729     }
4730
4731     return;
4732 }
4733
4734 sub vmconfig_delete_or_detach_drive {
4735     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4736
4737     my $drive = parse_drive($opt, $conf->{$opt});
4738
4739     my $rpcenv = PVE::RPCEnvironment::get();
4740     my $authuser = $rpcenv->get_user();
4741
4742     if ($force) {
4743         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4744         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4745     } else {
4746         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4747     }
4748 }
4749
4750
4751
4752 sub vmconfig_apply_pending {
4753     my ($vmid, $conf, $storecfg, $errors) = @_;
4754
4755     my $add_apply_error = sub {
4756         my ($opt, $msg) = @_;
4757         my $err_msg = "unable to apply pending change $opt : $msg";
4758         $errors->{$opt} = $err_msg;
4759         warn $err_msg;
4760     };
4761
4762     # cold plug
4763
4764     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4765     foreach my $opt (sort keys %$pending_delete_hash) {
4766         my $force = $pending_delete_hash->{$opt}->{force};
4767         eval {
4768             if ($opt =~ m/^unused/) {
4769                 die "internal error";
4770             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4771                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4772             }
4773         };
4774         if (my $err = $@) {
4775             $add_apply_error->($opt, $err);
4776         } else {
4777             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4778             delete $conf->{$opt};
4779         }
4780     }
4781
4782     PVE::QemuConfig->cleanup_pending($conf);
4783
4784     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4785         next if $opt eq 'delete'; # just to be sure
4786         eval {
4787             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4788                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4789             }
4790         };
4791         if (my $err = $@) {
4792             $add_apply_error->($opt, $err);
4793         } else {
4794             $conf->{$opt} = delete $conf->{pending}->{$opt};
4795         }
4796     }
4797
4798     # write all changes at once to avoid unnecessary i/o
4799     PVE::QemuConfig->write_config($vmid, $conf);
4800 }
4801
4802 sub vmconfig_update_net {
4803     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4804
4805     my $newnet = parse_net($value);
4806
4807     if ($conf->{$opt}) {
4808         my $oldnet = parse_net($conf->{$opt});
4809
4810         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4811             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4812             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4813             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4814
4815             # for non online change, we try to hot-unplug
4816             die "skip\n" if !$hotplug;
4817             vm_deviceunplug($vmid, $conf, $opt);
4818         } else {
4819
4820             die "internal error" if $opt !~ m/net(\d+)/;
4821             my $iface = "tap${vmid}i$1";
4822
4823             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4824                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4825                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4826                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4827                 PVE::Network::tap_unplug($iface);
4828
4829                 if ($have_sdn) {
4830                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4831                 } else {
4832                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4833                 }
4834             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4835                 # Rate can be applied on its own but any change above needs to
4836                 # include the rate in tap_plug since OVS resets everything.
4837                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4838             }
4839
4840             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4841                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4842             }
4843
4844             return 1;
4845         }
4846     }
4847
4848     if ($hotplug) {
4849         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4850     } else {
4851         die "skip\n";
4852     }
4853 }
4854
4855 sub vmconfig_update_disk {
4856     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4857
4858     my $drive = parse_drive($opt, $value);
4859
4860     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4861         my $media = $drive->{media} || 'disk';
4862         my $oldmedia = $old_drive->{media} || 'disk';
4863         die "unable to change media type\n" if $media ne $oldmedia;
4864
4865         if (!drive_is_cdrom($old_drive)) {
4866
4867             if ($drive->{file} ne $old_drive->{file}) {
4868
4869                 die "skip\n" if !$hotplug;
4870
4871                 # unplug and register as unused
4872                 vm_deviceunplug($vmid, $conf, $opt);
4873                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4874
4875             } else {
4876                 # update existing disk
4877
4878                 # skip non hotpluggable value
4879                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4880                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4881                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4882                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4883                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4884                     die "skip\n";
4885                 }
4886
4887                 # apply throttle
4888                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4889                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4890                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4891                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4892                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4893                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4894                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4895                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4896                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4897                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4898                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4899                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4900                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4901                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4902                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4903                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4904                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4905                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4906
4907                     qemu_block_set_io_throttle(
4908                         $vmid,"drive-$opt",
4909                         ($drive->{mbps} || 0)*1024*1024,
4910                         ($drive->{mbps_rd} || 0)*1024*1024,
4911                         ($drive->{mbps_wr} || 0)*1024*1024,
4912                          $drive->{iops} || 0,
4913                          $drive->{iops_rd} || 0,
4914                          $drive->{iops_wr} || 0,
4915                         ($drive->{mbps_max} || 0)*1024*1024,
4916                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4917                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4918                          $drive->{iops_max} || 0,
4919                          $drive->{iops_rd_max} || 0,
4920                          $drive->{iops_wr_max} || 0,
4921                          $drive->{bps_max_length} || 1,
4922                          $drive->{bps_rd_max_length} || 1,
4923                          $drive->{bps_wr_max_length} || 1,
4924                          $drive->{iops_max_length} || 1,
4925                          $drive->{iops_rd_max_length} || 1,
4926                          $drive->{iops_wr_max_length} || 1,
4927                     );
4928
4929                 }
4930
4931                 return 1;
4932             }
4933
4934         } else { # cdrom
4935
4936             if ($drive->{file} eq 'none') {
4937                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4938                 if (drive_is_cloudinit($old_drive)) {
4939                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4940                 }
4941             } else {
4942                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4943
4944                 # force eject if locked
4945                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4946
4947                 if ($path) {
4948                     mon_cmd($vmid, "blockdev-change-medium",
4949                         id => "$opt", filename => "$path");
4950                 }
4951             }
4952
4953             return 1;
4954         }
4955     }
4956
4957     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4958     # hotplug new disks
4959     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4960     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4961 }
4962
4963 # called in locked context by incoming migration
4964 sub vm_migrate_get_nbd_disks {
4965     my ($storecfg, $conf, $replicated_volumes) = @_;
4966
4967     my $local_volumes = {};
4968     PVE::QemuConfig->foreach_volume($conf, sub {
4969         my ($ds, $drive) = @_;
4970
4971         return if drive_is_cdrom($drive);
4972
4973         my $volid = $drive->{file};
4974
4975         return if !$volid;
4976
4977         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4978
4979         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4980         return if $scfg->{shared};
4981
4982         # replicated disks re-use existing state via bitmap
4983         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4984         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4985     });
4986     return $local_volumes;
4987 }
4988
4989 # called in locked context by incoming migration
4990 sub vm_migrate_alloc_nbd_disks {
4991     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4992
4993     my $format = undef;
4994
4995     my $nbd = {};
4996     foreach my $opt (sort keys %$source_volumes) {
4997         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4998
4999         if ($use_existing) {
5000             $nbd->{$opt}->{drivestr} = print_drive($drive);
5001             $nbd->{$opt}->{volid} = $volid;
5002             $nbd->{$opt}->{replicated} = 1;
5003             next;
5004         }
5005
5006         # If a remote storage is specified and the format of the original
5007         # volume is not available there, fall back to the default format.
5008         # Otherwise use the same format as the original.
5009         if (!$storagemap->{identity}) {
5010             $storeid = map_storage($storagemap, $storeid);
5011             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5012             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5013             my $fileFormat = qemu_img_format($scfg, $volname);
5014             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5015         } else {
5016             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5017             $format = qemu_img_format($scfg, $volname);
5018         }
5019
5020         my $size = $drive->{size} / 1024;
5021         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5022         my $newdrive = $drive;
5023         $newdrive->{format} = $format;
5024         $newdrive->{file} = $newvolid;
5025         my $drivestr = print_drive($newdrive);
5026         $nbd->{$opt}->{drivestr} = $drivestr;
5027         $nbd->{$opt}->{volid} = $newvolid;
5028     }
5029
5030     return $nbd;
5031 }
5032
5033 # see vm_start_nolock for parameters, additionally:
5034 # migrate_opts:
5035 #   storagemap = parsed storage map for allocating NBD disks
5036 sub vm_start {
5037     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5038
5039     return PVE::QemuConfig->lock_config($vmid, sub {
5040         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5041
5042         die "you can't start a vm if it's a template\n"
5043             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5044
5045         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5046         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5047
5048         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5049
5050         if ($has_backup_lock && $running) {
5051             # a backup is currently running, attempt to start the guest in the
5052             # existing QEMU instance
5053             return vm_resume($vmid);
5054         }
5055
5056         PVE::QemuConfig->check_lock($conf)
5057             if !($params->{skiplock} || $has_suspended_lock);
5058
5059         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5060
5061         die "VM $vmid already running\n" if $running;
5062
5063         if (my $storagemap = $migrate_opts->{storagemap}) {
5064             my $replicated = $migrate_opts->{replicated_volumes};
5065             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5066             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5067
5068             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5069                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5070             }
5071         }
5072
5073         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5074     });
5075 }
5076
5077
5078 # params:
5079 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5080 #   skiplock => 0/1, skip checking for config lock
5081 #   skiptemplate => 0/1, skip checking whether VM is template
5082 #   forcemachine => to force Qemu machine (rollback/migration)
5083 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5084 #   timeout => in seconds
5085 #   paused => start VM in paused state (backup)
5086 #   resume => resume from hibernation
5087 #   pbs-backing => {
5088 #      sata0 => {
5089 #         repository
5090 #         snapshot
5091 #         keyfile
5092 #         archive
5093 #      },
5094 #      virtio2 => ...
5095 #   }
5096 # migrate_opts:
5097 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5098 #   migratedfrom => source node
5099 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5100 #   network => CIDR of migration network
5101 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5102 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5103 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5104 sub vm_start_nolock {
5105     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5106
5107     my $statefile = $params->{statefile};
5108     my $resume = $params->{resume};
5109
5110     my $migratedfrom = $migrate_opts->{migratedfrom};
5111     my $migration_type = $migrate_opts->{type};
5112
5113     my $res = {};
5114
5115     # clean up leftover reboot request files
5116     eval { clear_reboot_request($vmid); };
5117     warn $@ if $@;
5118
5119     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5120         vmconfig_apply_pending($vmid, $conf, $storecfg);
5121         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5122     }
5123
5124     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5125
5126     my $defaults = load_defaults();
5127
5128     # set environment variable useful inside network script
5129     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5130
5131     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5132
5133     my $forcemachine = $params->{forcemachine};
5134     my $forcecpu = $params->{forcecpu};
5135     if ($resume) {
5136         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5137         $forcemachine = $conf->{runningmachine};
5138         $forcecpu = $conf->{runningcpu};
5139         print "Resuming suspended VM\n";
5140     }
5141
5142     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5143         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5144
5145     my $migration_ip;
5146     my $get_migration_ip = sub {
5147         my ($nodename) = @_;
5148
5149         return $migration_ip if defined($migration_ip);
5150
5151         my $cidr = $migrate_opts->{network};
5152
5153         if (!defined($cidr)) {
5154             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5155             $cidr = $dc_conf->{migration}->{network};
5156         }
5157
5158         if (defined($cidr)) {
5159             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5160
5161             die "could not get IP: no address configured on local " .
5162                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5163
5164             die "could not get IP: multiple addresses configured on local " .
5165                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5166
5167             $migration_ip = @$ips[0];
5168         }
5169
5170         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5171             if !defined($migration_ip);
5172
5173         return $migration_ip;
5174     };
5175
5176     my $migrate_uri;
5177     if ($statefile) {
5178         if ($statefile eq 'tcp') {
5179             my $localip = "localhost";
5180             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5181             my $nodename = nodename();
5182
5183             if (!defined($migration_type)) {
5184                 if (defined($datacenterconf->{migration}->{type})) {
5185                     $migration_type = $datacenterconf->{migration}->{type};
5186                 } else {
5187                     $migration_type = 'secure';
5188                 }
5189             }
5190
5191             if ($migration_type eq 'insecure') {
5192                 $localip = $get_migration_ip->($nodename);
5193                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5194             }
5195
5196             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5197             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5198             $migrate_uri = "tcp:${localip}:${migrate_port}";
5199             push @$cmd, '-incoming', $migrate_uri;
5200             push @$cmd, '-S';
5201
5202         } elsif ($statefile eq 'unix') {
5203             # should be default for secure migrations as a ssh TCP forward
5204             # tunnel is not deterministic reliable ready and fails regurarly
5205             # to set up in time, so use UNIX socket forwards
5206             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5207             unlink $socket_addr;
5208
5209             $migrate_uri = "unix:$socket_addr";
5210
5211             push @$cmd, '-incoming', $migrate_uri;
5212             push @$cmd, '-S';
5213
5214         } elsif (-e $statefile) {
5215             push @$cmd, '-loadstate', $statefile;
5216         } else {
5217             my $statepath = PVE::Storage::path($storecfg, $statefile);
5218             push @$vollist, $statefile;
5219             push @$cmd, '-loadstate', $statepath;
5220         }
5221     } elsif ($params->{paused}) {
5222         push @$cmd, '-S';
5223     }
5224
5225     # host pci devices
5226     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5227       my $d = parse_hostpci($conf->{"hostpci$i"});
5228       next if !$d;
5229       my $pcidevices = $d->{pciid};
5230       foreach my $pcidevice (@$pcidevices) {
5231             my $pciid = $pcidevice->{id};
5232
5233             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5234             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5235             die "no pci device info for device '$pciid'\n" if !$info;
5236
5237             if ($d->{mdev}) {
5238                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5239                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5240             } else {
5241                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5242                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5243                 die "can't reset PCI device '$pciid'\n"
5244                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5245             }
5246       }
5247     }
5248
5249     PVE::Storage::activate_volumes($storecfg, $vollist);
5250
5251     eval {
5252         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5253             outfunc => sub {}, errfunc => sub {});
5254     };
5255     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5256     # timeout should be more than enough here...
5257     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5258
5259     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5260                                               : $defaults->{cpuunits};
5261
5262     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5263     my %run_params = (
5264         timeout => $statefile ? undef : $start_timeout,
5265         umask => 0077,
5266         noerr => 1,
5267     );
5268
5269     # when migrating, prefix QEMU output so other side can pick up any
5270     # errors that might occur and show the user
5271     if ($migratedfrom) {
5272         $run_params{quiet} = 1;
5273         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5274     }
5275
5276     my %properties = (
5277         Slice => 'qemu.slice',
5278         KillMode => 'none'
5279     );
5280
5281     if (PVE::CGroup::cgroup_mode() == 2) {
5282         $properties{CPUWeight} = $cpuunits;
5283     } else {
5284         $properties{CPUShares} = $cpuunits;
5285     }
5286
5287     if (my $cpulimit = $conf->{cpulimit}) {
5288         $properties{CPUQuota} = int($cpulimit * 100);
5289     }
5290     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5291
5292     my $run_qemu = sub {
5293         PVE::Tools::run_fork sub {
5294             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5295
5296             my $exitcode = run_command($cmd, %run_params);
5297             die "QEMU exited with code $exitcode\n" if $exitcode;
5298         };
5299     };
5300
5301     if ($conf->{hugepages}) {
5302
5303         my $code = sub {
5304             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5305             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5306
5307             PVE::QemuServer::Memory::hugepages_mount();
5308             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5309
5310             eval { $run_qemu->() };
5311             if (my $err = $@) {
5312                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5313                     if !$conf->{keephugepages};
5314                 die $err;
5315             }
5316
5317             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5318                 if !$conf->{keephugepages};
5319         };
5320         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5321
5322     } else {
5323         eval { $run_qemu->() };
5324     }
5325
5326     if (my $err = $@) {
5327         # deactivate volumes if start fails
5328         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5329         die "start failed: $err";
5330     }
5331
5332     print "migration listens on $migrate_uri\n" if $migrate_uri;
5333     $res->{migrate_uri} = $migrate_uri;
5334
5335     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5336         eval { mon_cmd($vmid, "cont"); };
5337         warn $@ if $@;
5338     }
5339
5340     #start nbd server for storage migration
5341     if (my $nbd = $migrate_opts->{nbd}) {
5342         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5343
5344         my $migrate_storage_uri;
5345         # nbd_protocol_version > 0 for unix socket support
5346         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5347             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5348             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5349             $migrate_storage_uri = "nbd:unix:$socket_path";
5350         } else {
5351             my $nodename = nodename();
5352             my $localip = $get_migration_ip->($nodename);
5353             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5354             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5355
5356             mon_cmd($vmid, "nbd-server-start", addr => {
5357                 type => 'inet',
5358                 data => {
5359                     host => "${localip}",
5360                     port => "${storage_migrate_port}",
5361                 },
5362             });
5363             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5364             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5365         }
5366
5367         $res->{migrate_storage_uri} = $migrate_storage_uri;
5368
5369         foreach my $opt (sort keys %$nbd) {
5370             my $drivestr = $nbd->{$opt}->{drivestr};
5371             my $volid = $nbd->{$opt}->{volid};
5372             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5373             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5374             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5375             print "re-using replicated volume: $opt - $volid\n"
5376                 if $nbd->{$opt}->{replicated};
5377
5378             $res->{drives}->{$opt} = $nbd->{$opt};
5379             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5380         }
5381     }
5382
5383     if ($migratedfrom) {
5384         eval {
5385             set_migration_caps($vmid);
5386         };
5387         warn $@ if $@;
5388
5389         if ($spice_port) {
5390             print "spice listens on port $spice_port\n";
5391             $res->{spice_port} = $spice_port;
5392             if ($migrate_opts->{spice_ticket}) {
5393                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5394                     $migrate_opts->{spice_ticket});
5395                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5396             }
5397         }
5398
5399     } else {
5400         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5401             if !$statefile && $conf->{balloon};
5402
5403         foreach my $opt (keys %$conf) {
5404             next if $opt !~  m/^net\d+$/;
5405             my $nicconf = parse_net($conf->{$opt});
5406             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5407         }
5408     }
5409
5410     mon_cmd($vmid, 'qom-set',
5411                 path => "machine/peripheral/balloon0",
5412                 property => "guest-stats-polling-interval",
5413                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5414
5415     if ($resume) {
5416         print "Resumed VM, removing state\n";
5417         if (my $vmstate = $conf->{vmstate}) {
5418             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5419             PVE::Storage::vdisk_free($storecfg, $vmstate);
5420         }
5421         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5422         PVE::QemuConfig->write_config($vmid, $conf);
5423     }
5424
5425     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5426
5427     return $res;
5428 }
5429
5430 sub vm_commandline {
5431     my ($storecfg, $vmid, $snapname) = @_;
5432
5433     my $conf = PVE::QemuConfig->load_config($vmid);
5434     my $forcemachine;
5435     my $forcecpu;
5436
5437     if ($snapname) {
5438         my $snapshot = $conf->{snapshots}->{$snapname};
5439         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5440
5441         # check for machine or CPU overrides in snapshot
5442         $forcemachine = $snapshot->{runningmachine};
5443         $forcecpu = $snapshot->{runningcpu};
5444
5445         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5446
5447         $conf = $snapshot;
5448     }
5449
5450     my $defaults = load_defaults();
5451
5452     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5453         $forcemachine, $forcecpu);
5454
5455     return PVE::Tools::cmd2string($cmd);
5456 }
5457
5458 sub vm_reset {
5459     my ($vmid, $skiplock) = @_;
5460
5461     PVE::QemuConfig->lock_config($vmid, sub {
5462
5463         my $conf = PVE::QemuConfig->load_config($vmid);
5464
5465         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5466
5467         mon_cmd($vmid, "system_reset");
5468     });
5469 }
5470
5471 sub get_vm_volumes {
5472     my ($conf) = @_;
5473
5474     my $vollist = [];
5475     foreach_volid($conf, sub {
5476         my ($volid, $attr) = @_;
5477
5478         return if $volid =~ m|^/|;
5479
5480         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5481         return if !$sid;
5482
5483         push @$vollist, $volid;
5484     });
5485
5486     return $vollist;
5487 }
5488
5489 sub vm_stop_cleanup {
5490     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5491
5492     eval {
5493
5494         if (!$keepActive) {
5495             my $vollist = get_vm_volumes($conf);
5496             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5497         }
5498
5499         foreach my $ext (qw(mon qmp pid vnc qga)) {
5500             unlink "/var/run/qemu-server/${vmid}.$ext";
5501         }
5502
5503         if ($conf->{ivshmem}) {
5504             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5505             # just delete it for now, VMs which have this already open do not
5506             # are affected, but new VMs will get a separated one. If this
5507             # becomes an issue we either add some sort of ref-counting or just
5508             # add a "don't delete on stop" flag to the ivshmem format.
5509             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5510         }
5511
5512         foreach my $key (keys %$conf) {
5513             next if $key !~ m/^hostpci(\d+)$/;
5514             my $hostpciindex = $1;
5515             my $d = parse_hostpci($conf->{$key});
5516             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5517
5518             foreach my $pci (@{$d->{pciid}}) {
5519                 my $pciid = $pci->{id};
5520                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5521             }
5522         }
5523
5524         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5525     };
5526     warn $@ if $@; # avoid errors - just warn
5527 }
5528
5529 # call only in locked context
5530 sub _do_vm_stop {
5531     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5532
5533     my $pid = check_running($vmid, $nocheck);
5534     return if !$pid;
5535
5536     my $conf;
5537     if (!$nocheck) {
5538         $conf = PVE::QemuConfig->load_config($vmid);
5539         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5540         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5541             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5542             $timeout = $opts->{down} if $opts->{down};
5543         }
5544         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5545     }
5546
5547     eval {
5548         if ($shutdown) {
5549             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5550                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5551             } else {
5552                 mon_cmd($vmid, "system_powerdown");
5553             }
5554         } else {
5555             mon_cmd($vmid, "quit");
5556         }
5557     };
5558     my $err = $@;
5559
5560     if (!$err) {
5561         $timeout = 60 if !defined($timeout);
5562
5563         my $count = 0;
5564         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5565             $count++;
5566             sleep 1;
5567         }
5568
5569         if ($count >= $timeout) {
5570             if ($force) {
5571                 warn "VM still running - terminating now with SIGTERM\n";
5572                 kill 15, $pid;
5573             } else {
5574                 die "VM quit/powerdown failed - got timeout\n";
5575             }
5576         } else {
5577             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5578             return;
5579         }
5580     } else {
5581        if (!check_running($vmid, $nocheck)) {
5582             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5583             return;
5584        }
5585        if ($force) {
5586             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5587             kill 15, $pid;
5588         } else {
5589             die "VM quit/powerdown failed\n";
5590         }
5591     }
5592
5593     # wait again
5594     $timeout = 10;
5595
5596     my $count = 0;
5597     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5598         $count++;
5599         sleep 1;
5600     }
5601
5602     if ($count >= $timeout) {
5603         warn "VM still running - terminating now with SIGKILL\n";
5604         kill 9, $pid;
5605         sleep 1;
5606     }
5607
5608     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5609 }
5610
5611 # Note: use $nocheck to skip tests if VM configuration file exists.
5612 # We need that when migration VMs to other nodes (files already moved)
5613 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5614 sub vm_stop {
5615     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5616
5617     $force = 1 if !defined($force) && !$shutdown;
5618
5619     if ($migratedfrom){
5620         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5621         kill 15, $pid if $pid;
5622         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5623         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5624         return;
5625     }
5626
5627     PVE::QemuConfig->lock_config($vmid, sub {
5628         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5629    });
5630 }
5631
5632 sub vm_reboot {
5633     my ($vmid, $timeout) = @_;
5634
5635     PVE::QemuConfig->lock_config($vmid, sub {
5636         eval {
5637
5638             # only reboot if running, as qmeventd starts it again on a stop event
5639             return if !check_running($vmid);
5640
5641             create_reboot_request($vmid);
5642
5643             my $storecfg = PVE::Storage::config();
5644             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5645
5646         };
5647         if (my $err = $@) {
5648             # avoid that the next normal shutdown will be confused for a reboot
5649             clear_reboot_request($vmid);
5650             die $err;
5651         }
5652    });
5653 }
5654
5655 # note: if using the statestorage parameter, the caller has to check privileges
5656 sub vm_suspend {
5657     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5658
5659     my $conf;
5660     my $path;
5661     my $storecfg;
5662     my $vmstate;
5663
5664     PVE::QemuConfig->lock_config($vmid, sub {
5665
5666         $conf = PVE::QemuConfig->load_config($vmid);
5667
5668         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5669         PVE::QemuConfig->check_lock($conf)
5670             if !($skiplock || $is_backing_up);
5671
5672         die "cannot suspend to disk during backup\n"
5673             if $is_backing_up && $includestate;
5674
5675         if ($includestate) {
5676             $conf->{lock} = 'suspending';
5677             my $date = strftime("%Y-%m-%d", localtime(time()));
5678             $storecfg = PVE::Storage::config();
5679             if (!$statestorage) {
5680                 $statestorage = find_vmstate_storage($conf, $storecfg);
5681                 # check permissions for the storage
5682                 my $rpcenv = PVE::RPCEnvironment::get();
5683                 if ($rpcenv->{type} ne 'cli') {
5684                     my $authuser = $rpcenv->get_user();
5685                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5686                 }
5687             }
5688
5689
5690             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5691                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5692             $path = PVE::Storage::path($storecfg, $vmstate);
5693             PVE::QemuConfig->write_config($vmid, $conf);
5694         } else {
5695             mon_cmd($vmid, "stop");
5696         }
5697     });
5698
5699     if ($includestate) {
5700         # save vm state
5701         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5702
5703         eval {
5704             set_migration_caps($vmid, 1);
5705             mon_cmd($vmid, "savevm-start", statefile => $path);
5706             for(;;) {
5707                 my $state = mon_cmd($vmid, "query-savevm");
5708                 if (!$state->{status}) {
5709                     die "savevm not active\n";
5710                 } elsif ($state->{status} eq 'active') {
5711                     sleep(1);
5712                     next;
5713                 } elsif ($state->{status} eq 'completed') {
5714                     print "State saved, quitting\n";
5715                     last;
5716                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5717                     die "query-savevm failed with error '$state->{error}'\n"
5718                 } else {
5719                     die "query-savevm returned status '$state->{status}'\n";
5720                 }
5721             }
5722         };
5723         my $err = $@;
5724
5725         PVE::QemuConfig->lock_config($vmid, sub {
5726             $conf = PVE::QemuConfig->load_config($vmid);
5727             if ($err) {
5728                 # cleanup, but leave suspending lock, to indicate something went wrong
5729                 eval {
5730                     mon_cmd($vmid, "savevm-end");
5731                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5732                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5733                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5734                     PVE::QemuConfig->write_config($vmid, $conf);
5735                 };
5736                 warn $@ if $@;
5737                 die $err;
5738             }
5739
5740             die "lock changed unexpectedly\n"
5741                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5742
5743             mon_cmd($vmid, "quit");
5744             $conf->{lock} = 'suspended';
5745             PVE::QemuConfig->write_config($vmid, $conf);
5746         });
5747     }
5748 }
5749
5750 sub vm_resume {
5751     my ($vmid, $skiplock, $nocheck) = @_;
5752
5753     PVE::QemuConfig->lock_config($vmid, sub {
5754         my $res = mon_cmd($vmid, 'query-status');
5755         my $resume_cmd = 'cont';
5756         my $reset = 0;
5757
5758         if ($res->{status}) {
5759             return if $res->{status} eq 'running'; # job done, go home
5760             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5761             $reset = 1 if $res->{status} eq 'shutdown';
5762         }
5763
5764         if (!$nocheck) {
5765
5766             my $conf = PVE::QemuConfig->load_config($vmid);
5767
5768             PVE::QemuConfig->check_lock($conf)
5769                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5770         }
5771
5772         if ($reset) {
5773             # required if a VM shuts down during a backup and we get a resume
5774             # request before the backup finishes for example
5775             mon_cmd($vmid, "system_reset");
5776         }
5777         mon_cmd($vmid, $resume_cmd);
5778     });
5779 }
5780
5781 sub vm_sendkey {
5782     my ($vmid, $skiplock, $key) = @_;
5783
5784     PVE::QemuConfig->lock_config($vmid, sub {
5785
5786         my $conf = PVE::QemuConfig->load_config($vmid);
5787
5788         # there is no qmp command, so we use the human monitor command
5789         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5790         die $res if $res ne '';
5791     });
5792 }
5793
5794 # vzdump restore implementaion
5795
5796 sub tar_archive_read_firstfile {
5797     my $archive = shift;
5798
5799     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5800
5801     # try to detect archive type first
5802     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5803         die "unable to open file '$archive'\n";
5804     my $firstfile = <$fh>;
5805     kill 15, $pid;
5806     close $fh;
5807
5808     die "ERROR: archive contaions no data\n" if !$firstfile;
5809     chomp $firstfile;
5810
5811     return $firstfile;
5812 }
5813
5814 sub tar_restore_cleanup {
5815     my ($storecfg, $statfile) = @_;
5816
5817     print STDERR "starting cleanup\n";
5818
5819     if (my $fd = IO::File->new($statfile, "r")) {
5820         while (defined(my $line = <$fd>)) {
5821             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5822                 my $volid = $2;
5823                 eval {
5824                     if ($volid =~ m|^/|) {
5825                         unlink $volid || die 'unlink failed\n';
5826                     } else {
5827                         PVE::Storage::vdisk_free($storecfg, $volid);
5828                     }
5829                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5830                 };
5831                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5832             } else {
5833                 print STDERR "unable to parse line in statfile - $line";
5834             }
5835         }
5836         $fd->close();
5837     }
5838 }
5839
5840 sub restore_file_archive {
5841     my ($archive, $vmid, $user, $opts) = @_;
5842
5843     return restore_vma_archive($archive, $vmid, $user, $opts)
5844         if $archive eq '-';
5845
5846     my $info = PVE::Storage::archive_info($archive);
5847     my $format = $opts->{format} // $info->{format};
5848     my $comp = $info->{compression};
5849
5850     # try to detect archive format
5851     if ($format eq 'tar') {
5852         return restore_tar_archive($archive, $vmid, $user, $opts);
5853     } else {
5854         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5855     }
5856 }
5857
5858 # hepler to remove disks that will not be used after restore
5859 my $restore_cleanup_oldconf = sub {
5860     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5861
5862     PVE::QemuConfig->foreach_volume($oldconf, sub {
5863         my ($ds, $drive) = @_;
5864
5865         return if drive_is_cdrom($drive, 1);
5866
5867         my $volid = $drive->{file};
5868         return if !$volid || $volid =~ m|^/|;
5869
5870         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5871         return if !$path || !$owner || ($owner != $vmid);
5872
5873         # Note: only delete disk we want to restore
5874         # other volumes will become unused
5875         if ($virtdev_hash->{$ds}) {
5876             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5877             if (my $err = $@) {
5878                 warn $err;
5879             }
5880         }
5881     });
5882
5883     # delete vmstate files, after the restore we have no snapshots anymore
5884     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5885         my $snap = $oldconf->{snapshots}->{$snapname};
5886         if ($snap->{vmstate}) {
5887             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5888             if (my $err = $@) {
5889                 warn $err;
5890             }
5891         }
5892     }
5893 };
5894
5895 # Helper to parse vzdump backup device hints
5896 #
5897 # $rpcenv: Environment, used to ckeck storage permissions
5898 # $user: User ID, to check storage permissions
5899 # $storecfg: Storage configuration
5900 # $fh: the file handle for reading the configuration
5901 # $devinfo: should contain device sizes for all backu-up'ed devices
5902 # $options: backup options (pool, default storage)
5903 #
5904 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5905 my $parse_backup_hints = sub {
5906     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5907
5908     my $virtdev_hash = {};
5909
5910     while (defined(my $line = <$fh>)) {
5911         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5912             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5913             die "archive does not contain data for drive '$virtdev'\n"
5914                 if !$devinfo->{$devname};
5915
5916             if (defined($options->{storage})) {
5917                 $storeid = $options->{storage} || 'local';
5918             } elsif (!$storeid) {
5919                 $storeid = 'local';
5920             }
5921             $format = 'raw' if !$format;
5922             $devinfo->{$devname}->{devname} = $devname;
5923             $devinfo->{$devname}->{virtdev} = $virtdev;
5924             $devinfo->{$devname}->{format} = $format;
5925             $devinfo->{$devname}->{storeid} = $storeid;
5926
5927             # check permission on storage
5928             my $pool = $options->{pool}; # todo: do we need that?
5929             if ($user ne 'root@pam') {
5930                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5931             }
5932
5933             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5934         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5935             my $virtdev = $1;
5936             my $drive = parse_drive($virtdev, $2);
5937             if (drive_is_cloudinit($drive)) {
5938                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5939                 $storeid = $options->{storage} if defined ($options->{storage});
5940                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5941                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5942
5943                 $virtdev_hash->{$virtdev} = {
5944                     format => $format,
5945                     storeid => $storeid,
5946                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5947                     is_cloudinit => 1,
5948                 };
5949             }
5950         }
5951     }
5952
5953     return $virtdev_hash;
5954 };
5955
5956 # Helper to allocate and activate all volumes required for a restore
5957 #
5958 # $storecfg: Storage configuration
5959 # $virtdev_hash: as returned by parse_backup_hints()
5960 #
5961 # Returns: { $virtdev => $volid }
5962 my $restore_allocate_devices = sub {
5963     my ($storecfg, $virtdev_hash, $vmid) = @_;
5964
5965     my $map = {};
5966     foreach my $virtdev (sort keys %$virtdev_hash) {
5967         my $d = $virtdev_hash->{$virtdev};
5968         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5969         my $storeid = $d->{storeid};
5970         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5971
5972         # test if requested format is supported
5973         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5974         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5975         $d->{format} = $defFormat if !$supported;
5976
5977         my $name;
5978         if ($d->{is_cloudinit}) {
5979             $name = "vm-$vmid-cloudinit";
5980             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5981             if ($scfg->{path}) {
5982                 $name .= ".$d->{format}";
5983             }
5984         }
5985
5986         my $volid = PVE::Storage::vdisk_alloc(
5987             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5988
5989         print STDERR "new volume ID is '$volid'\n";
5990         $d->{volid} = $volid;
5991
5992         PVE::Storage::activate_volumes($storecfg, [$volid]);
5993
5994         $map->{$virtdev} = $volid;
5995     }
5996
5997     return $map;
5998 };
5999
6000 sub restore_update_config_line {
6001     my ($cookie, $map, $line, $unique) = @_;
6002
6003     return '' if $line =~ m/^\#qmdump\#/;
6004     return '' if $line =~ m/^\#vzdump\#/;
6005     return '' if $line =~ m/^lock:/;
6006     return '' if $line =~ m/^unused\d+:/;
6007     return '' if $line =~ m/^parent:/;
6008
6009     my $res = '';
6010
6011     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6012     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6013         # try to convert old 1.X settings
6014         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6015         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6016             my ($model, $macaddr) = split(/\=/, $devconfig);
6017             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6018             my $net = {
6019                 model => $model,
6020                 bridge => "vmbr$ind",
6021                 macaddr => $macaddr,
6022             };
6023             my $netstr = print_net($net);
6024
6025             $res .= "net$cookie->{netcount}: $netstr\n";
6026             $cookie->{netcount}++;
6027         }
6028     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6029         my ($id, $netstr) = ($1, $2);
6030         my $net = parse_net($netstr);
6031         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6032         $netstr = print_net($net);
6033         $res .= "$id: $netstr\n";
6034     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6035         my $virtdev = $1;
6036         my $value = $3;
6037         my $di = parse_drive($virtdev, $value);
6038         if (defined($di->{backup}) && !$di->{backup}) {
6039             $res .= "#$line";
6040         } elsif ($map->{$virtdev}) {
6041             delete $di->{format}; # format can change on restore
6042             $di->{file} = $map->{$virtdev};
6043             $value = print_drive($di);
6044             $res .= "$virtdev: $value\n";
6045         } else {
6046             $res .= $line;
6047         }
6048     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6049         my $vmgenid = $1;
6050         if ($vmgenid ne '0') {
6051             # always generate a new vmgenid if there was a valid one setup
6052             $vmgenid = generate_uuid();
6053         }
6054         $res .= "vmgenid: $vmgenid\n";
6055     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6056         my ($uuid, $uuid_str);
6057         UUID::generate($uuid);
6058         UUID::unparse($uuid, $uuid_str);
6059         my $smbios1 = parse_smbios1($2);
6060         $smbios1->{uuid} = $uuid_str;
6061         $res .= $1.print_smbios1($smbios1)."\n";
6062     } else {
6063         $res .= $line;
6064     }
6065
6066     return $res;
6067 }
6068
6069 my $restore_deactivate_volumes = sub {
6070     my ($storecfg, $devinfo) = @_;
6071
6072     my $vollist = [];
6073     foreach my $devname (keys %$devinfo) {
6074         my $volid = $devinfo->{$devname}->{volid};
6075         push @$vollist, $volid if $volid;
6076     }
6077
6078     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6079 };
6080
6081 my $restore_destroy_volumes = sub {
6082     my ($storecfg, $devinfo) = @_;
6083
6084     foreach my $devname (keys %$devinfo) {
6085         my $volid = $devinfo->{$devname}->{volid};
6086         next if !$volid;
6087         eval {
6088             if ($volid =~ m|^/|) {
6089                 unlink $volid || die 'unlink failed\n';
6090             } else {
6091                 PVE::Storage::vdisk_free($storecfg, $volid);
6092             }
6093             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6094         };
6095         print STDERR "unable to cleanup '$volid' - $@" if $@;
6096     }
6097 };
6098
6099 sub scan_volids {
6100     my ($cfg, $vmid) = @_;
6101
6102     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6103
6104     my $volid_hash = {};
6105     foreach my $storeid (keys %$info) {
6106         foreach my $item (@{$info->{$storeid}}) {
6107             next if !($item->{volid} && $item->{size});
6108             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6109             $volid_hash->{$item->{volid}} = $item;
6110         }
6111     }
6112
6113     return $volid_hash;
6114 }
6115
6116 sub update_disk_config {
6117     my ($vmid, $conf, $volid_hash) = @_;
6118
6119     my $changes;
6120     my $prefix = "VM $vmid";
6121
6122     # used and unused disks
6123     my $referenced = {};
6124
6125     # Note: it is allowed to define multiple storages with same path (alias), so
6126     # we need to check both 'volid' and real 'path' (two different volid can point
6127     # to the same path).
6128
6129     my $referencedpath = {};
6130
6131     # update size info
6132     PVE::QemuConfig->foreach_volume($conf, sub {
6133         my ($opt, $drive) = @_;
6134
6135         my $volid = $drive->{file};
6136         return if !$volid;
6137         my $volume = $volid_hash->{$volid};
6138
6139         # mark volid as "in-use" for next step
6140         $referenced->{$volid} = 1;
6141         if ($volume && (my $path = $volume->{path})) {
6142             $referencedpath->{$path} = 1;
6143         }
6144
6145         return if drive_is_cdrom($drive);
6146         return if !$volume;
6147
6148         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6149         if (defined($updated)) {
6150             $changes = 1;
6151             $conf->{$opt} = print_drive($updated);
6152             print "$prefix ($opt): $msg\n";
6153         }
6154     });
6155
6156     # remove 'unusedX' entry if volume is used
6157     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6158         my ($opt, $drive) = @_;
6159
6160         my $volid = $drive->{file};
6161         return if !$volid;
6162
6163         my $path;
6164         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6165         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6166             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6167             $changes = 1;
6168             delete $conf->{$opt};
6169         }
6170
6171         $referenced->{$volid} = 1;
6172         $referencedpath->{$path} = 1 if $path;
6173     });
6174
6175     foreach my $volid (sort keys %$volid_hash) {
6176         next if $volid =~ m/vm-$vmid-state-/;
6177         next if $referenced->{$volid};
6178         my $path = $volid_hash->{$volid}->{path};
6179         next if !$path; # just to be sure
6180         next if $referencedpath->{$path};
6181         $changes = 1;
6182         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6183         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6184         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6185     }
6186
6187     return $changes;
6188 }
6189
6190 sub rescan {
6191     my ($vmid, $nolock, $dryrun) = @_;
6192
6193     my $cfg = PVE::Storage::config();
6194
6195     print "rescan volumes...\n";
6196     my $volid_hash = scan_volids($cfg, $vmid);
6197
6198     my $updatefn =  sub {
6199         my ($vmid) = @_;
6200
6201         my $conf = PVE::QemuConfig->load_config($vmid);
6202
6203         PVE::QemuConfig->check_lock($conf);
6204
6205         my $vm_volids = {};
6206         foreach my $volid (keys %$volid_hash) {
6207             my $info = $volid_hash->{$volid};
6208             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6209         }
6210
6211         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6212
6213         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6214     };
6215
6216     if (defined($vmid)) {
6217         if ($nolock) {
6218             &$updatefn($vmid);
6219         } else {
6220             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6221         }
6222     } else {
6223         my $vmlist = config_list();
6224         foreach my $vmid (keys %$vmlist) {
6225             if ($nolock) {
6226                 &$updatefn($vmid);
6227             } else {
6228                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6229             }
6230         }
6231     }
6232 }
6233
6234 sub restore_proxmox_backup_archive {
6235     my ($archive, $vmid, $user, $options) = @_;
6236
6237     my $storecfg = PVE::Storage::config();
6238
6239     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6240     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6241
6242     my $fingerprint = $scfg->{fingerprint};
6243     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6244
6245     my $repo = PVE::PBSClient::get_repository($scfg);
6246
6247     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6248     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6249     local $ENV{PBS_PASSWORD} = $password;
6250     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6251
6252     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6253         PVE::Storage::parse_volname($storecfg, $archive);
6254
6255     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6256
6257     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6258
6259     my $tmpdir = "/var/tmp/vzdumptmp$$";
6260     rmtree $tmpdir;
6261     mkpath $tmpdir;
6262
6263     my $conffile = PVE::QemuConfig->config_file($vmid);
6264      # disable interrupts (always do cleanups)
6265     local $SIG{INT} =
6266         local $SIG{TERM} =
6267         local $SIG{QUIT} =
6268         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6269
6270     # Note: $oldconf is undef if VM does not exists
6271     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6272     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6273     my $new_conf_raw = '';
6274
6275     my $rpcenv = PVE::RPCEnvironment::get();
6276     my $devinfo = {};
6277
6278     eval {
6279         # enable interrupts
6280         local $SIG{INT} =
6281             local $SIG{TERM} =
6282             local $SIG{QUIT} =
6283             local $SIG{HUP} =
6284             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6285
6286         my $cfgfn = "$tmpdir/qemu-server.conf";
6287         my $firewall_config_fn = "$tmpdir/fw.conf";
6288         my $index_fn = "$tmpdir/index.json";
6289
6290         my $cmd = "restore";
6291
6292         my $param = [$pbs_backup_name, "index.json", $index_fn];
6293         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6294         my $index = PVE::Tools::file_get_contents($index_fn);
6295         $index = decode_json($index);
6296
6297         # print Dumper($index);
6298         foreach my $info (@{$index->{files}}) {
6299             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6300                 my $devname = $1;
6301                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6302                     $devinfo->{$devname}->{size} = $1;
6303                 } else {
6304                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6305                 }
6306             }
6307         }
6308
6309         my $is_qemu_server_backup = scalar(
6310             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6311         );
6312         if (!$is_qemu_server_backup) {
6313             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6314         }
6315         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6316
6317         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6318         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6319
6320         if ($has_firewall_config) {
6321             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6322             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6323
6324             my $pve_firewall_dir = '/etc/pve/firewall';
6325             mkdir $pve_firewall_dir; # make sure the dir exists
6326             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6327         }
6328
6329         my $fh = IO::File->new($cfgfn, "r") ||
6330             die "unable to read qemu-server.conf - $!\n";
6331
6332         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6333
6334         # fixme: rate limit?
6335
6336         # create empty/temp config
6337         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6338
6339         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6340
6341         # allocate volumes
6342         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6343
6344         if (!$options->{live}) {
6345             foreach my $virtdev (sort keys %$virtdev_hash) {
6346                 my $d = $virtdev_hash->{$virtdev};
6347                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6348
6349                 my $volid = $d->{volid};
6350
6351                 my $path = PVE::Storage::path($storecfg, $volid);
6352
6353                 my $pbs_restore_cmd = [
6354                     '/usr/bin/pbs-restore',
6355                     '--repository', $repo,
6356                     $pbs_backup_name,
6357                     "$d->{devname}.img.fidx",
6358                     $path,
6359                     '--verbose',
6360                     ];
6361
6362                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6363                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6364
6365                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6366                     push @$pbs_restore_cmd, '--skip-zero';
6367                 }
6368
6369                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6370                 print "restore proxmox backup image: $dbg_cmdstring\n";
6371                 run_command($pbs_restore_cmd);
6372             }
6373         }
6374
6375         $fh->seek(0, 0) || die "seek failed - $!\n";
6376
6377         my $cookie = { netcount => 0 };
6378         while (defined(my $line = <$fh>)) {
6379             $new_conf_raw .= restore_update_config_line(
6380                 $cookie,
6381                 $map,
6382                 $line,
6383                 $options->{unique},
6384             );
6385         }
6386
6387         $fh->close();
6388     };
6389     my $err = $@;
6390
6391     if ($err || !$options->{live}) {
6392         $restore_deactivate_volumes->($storecfg, $devinfo);
6393     }
6394
6395     rmtree $tmpdir;
6396
6397     if ($err) {
6398         $restore_destroy_volumes->($storecfg, $devinfo);
6399         die $err;
6400     }
6401
6402     if ($options->{live}) {
6403         # keep lock during live-restore
6404         $new_conf_raw .= "\nlock: create";
6405     }
6406
6407     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6408
6409     PVE::Cluster::cfs_update(); # make sure we read new file
6410
6411     eval { rescan($vmid, 1); };
6412     warn $@ if $@;
6413
6414     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6415
6416     if ($options->{live}) {
6417         # enable interrupts
6418         local $SIG{INT} =
6419             local $SIG{TERM} =
6420             local $SIG{QUIT} =
6421             local $SIG{HUP} =
6422             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6423
6424         my $conf = PVE::QemuConfig->load_config($vmid);
6425         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6426
6427         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6428
6429         PVE::QemuConfig->remove_lock($vmid, "create");
6430     }
6431 }
6432
6433 sub pbs_live_restore {
6434     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6435
6436     print "starting VM for live-restore\n";
6437     print "repository: '$repo', snapshot: '$snap'\n";
6438
6439     my $pbs_backing = {};
6440     for my $ds (keys %$restored_disks) {
6441         $ds =~ m/^drive-(.*)$/;
6442         my $confname = $1;
6443         $pbs_backing->{$confname} = {
6444             repository => $repo,
6445             snapshot => $snap,
6446             archive => "$ds.img.fidx",
6447         };
6448         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6449
6450         my $drive = parse_drive($confname, $conf->{$confname});
6451         print "restoring '$ds' to '$drive->{file}'\n";
6452     }
6453
6454     my $drives_streamed = 0;
6455     eval {
6456         # make sure HA doesn't interrupt our restore by stopping the VM
6457         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6458             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6459         }
6460
6461         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6462         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6463         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6464
6465         my $qmeventd_fd = register_qmeventd_handle($vmid);
6466
6467         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6468         # this will effectively collapse the backing image chain consisting of
6469         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6470         # removes itself once all backing images vanish with 'auto-remove=on')
6471         my $jobs = {};
6472         for my $ds (sort keys %$restored_disks) {
6473             my $job_id = "restore-$ds";
6474             mon_cmd($vmid, 'block-stream',
6475                 'job-id' => $job_id,
6476                 device => "$ds",
6477             );
6478             $jobs->{$job_id} = {};
6479         }
6480
6481         mon_cmd($vmid, 'cont');
6482         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6483
6484         print "restore-drive jobs finished successfully, removing all tracking block devices"
6485             ." to disconnect from Proxmox Backup Server\n";
6486
6487         for my $ds (sort keys %$restored_disks) {
6488             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6489         }
6490
6491         close($qmeventd_fd);
6492     };
6493
6494     my $err = $@;
6495
6496     if ($err) {
6497         warn "An error occured during live-restore: $err\n";
6498         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6499         die "live-restore failed\n";
6500     }
6501 }
6502
6503 sub restore_vma_archive {
6504     my ($archive, $vmid, $user, $opts, $comp) = @_;
6505
6506     my $readfrom = $archive;
6507
6508     my $cfg = PVE::Storage::config();
6509     my $commands = [];
6510     my $bwlimit = $opts->{bwlimit};
6511
6512     my $dbg_cmdstring = '';
6513     my $add_pipe = sub {
6514         my ($cmd) = @_;
6515         push @$commands, $cmd;
6516         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6517         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6518         $readfrom = '-';
6519     };
6520
6521     my $input = undef;
6522     if ($archive eq '-') {
6523         $input = '<&STDIN';
6524     } else {
6525         # If we use a backup from a PVE defined storage we also consider that
6526         # storage's rate limit:
6527         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6528         if (defined($volid)) {
6529             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6530             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6531             if ($readlimit) {
6532                 print STDERR "applying read rate limit: $readlimit\n";
6533                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6534                 $add_pipe->($cstream);
6535             }
6536         }
6537     }
6538
6539     if ($comp) {
6540         my $info = PVE::Storage::decompressor_info('vma', $comp);
6541         my $cmd = $info->{decompressor};
6542         push @$cmd, $readfrom;
6543         $add_pipe->($cmd);
6544     }
6545
6546     my $tmpdir = "/var/tmp/vzdumptmp$$";
6547     rmtree $tmpdir;
6548
6549     # disable interrupts (always do cleanups)
6550     local $SIG{INT} =
6551         local $SIG{TERM} =
6552         local $SIG{QUIT} =
6553         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6554
6555     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6556     POSIX::mkfifo($mapfifo, 0600);
6557     my $fifofh;
6558     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6559
6560     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6561
6562     my $oldtimeout;
6563     my $timeout = 5;
6564
6565     my $devinfo = {};
6566
6567     my $rpcenv = PVE::RPCEnvironment::get();
6568
6569     my $conffile = PVE::QemuConfig->config_file($vmid);
6570
6571     # Note: $oldconf is undef if VM does not exist
6572     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6573     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6574     my $new_conf_raw = '';
6575
6576     my %storage_limits;
6577
6578     my $print_devmap = sub {
6579         my $cfgfn = "$tmpdir/qemu-server.conf";
6580
6581         # we can read the config - that is already extracted
6582         my $fh = IO::File->new($cfgfn, "r") ||
6583             die "unable to read qemu-server.conf - $!\n";
6584
6585         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6586         if (-f $fwcfgfn) {
6587             my $pve_firewall_dir = '/etc/pve/firewall';
6588             mkdir $pve_firewall_dir; # make sure the dir exists
6589             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6590         }
6591
6592         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6593
6594         foreach my $info (values %{$virtdev_hash}) {
6595             my $storeid = $info->{storeid};
6596             next if defined($storage_limits{$storeid});
6597
6598             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6599             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6600             $storage_limits{$storeid} = $limit * 1024;
6601         }
6602
6603         foreach my $devname (keys %$devinfo) {
6604             die "found no device mapping information for device '$devname'\n"
6605                 if !$devinfo->{$devname}->{virtdev};
6606         }
6607
6608         # create empty/temp config
6609         if ($oldconf) {
6610             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6611             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6612         }
6613
6614         # allocate volumes
6615         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6616
6617         # print restore information to $fifofh
6618         foreach my $virtdev (sort keys %$virtdev_hash) {
6619             my $d = $virtdev_hash->{$virtdev};
6620             next if $d->{is_cloudinit}; # no need to restore cloudinit
6621
6622             my $storeid = $d->{storeid};
6623             my $volid = $d->{volid};
6624
6625             my $map_opts = '';
6626             if (my $limit = $storage_limits{$storeid}) {
6627                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6628             }
6629
6630             my $write_zeros = 1;
6631             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6632                 $write_zeros = 0;
6633             }
6634
6635             my $path = PVE::Storage::path($cfg, $volid);
6636
6637             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6638
6639             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6640         }
6641
6642         $fh->seek(0, 0) || die "seek failed - $!\n";
6643
6644         my $cookie = { netcount => 0 };
6645         while (defined(my $line = <$fh>)) {
6646             $new_conf_raw .= restore_update_config_line(
6647                 $cookie,
6648                 $map,
6649                 $line,
6650                 $opts->{unique},
6651             );
6652         }
6653
6654         $fh->close();
6655     };
6656
6657     eval {
6658         # enable interrupts
6659         local $SIG{INT} =
6660             local $SIG{TERM} =
6661             local $SIG{QUIT} =
6662             local $SIG{HUP} =
6663             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6664         local $SIG{ALRM} = sub { die "got timeout\n"; };
6665
6666         $oldtimeout = alarm($timeout);
6667
6668         my $parser = sub {
6669             my $line = shift;
6670
6671             print "$line\n";
6672
6673             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6674                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6675                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6676             } elsif ($line =~ m/^CTIME: /) {
6677                 # we correctly received the vma config, so we can disable
6678                 # the timeout now for disk allocation (set to 10 minutes, so
6679                 # that we always timeout if something goes wrong)
6680                 alarm(600);
6681                 &$print_devmap();
6682                 print $fifofh "done\n";
6683                 my $tmp = $oldtimeout || 0;
6684                 $oldtimeout = undef;
6685                 alarm($tmp);
6686                 close($fifofh);
6687                 $fifofh = undef;
6688             }
6689         };
6690
6691         print "restore vma archive: $dbg_cmdstring\n";
6692         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6693     };
6694     my $err = $@;
6695
6696     alarm($oldtimeout) if $oldtimeout;
6697
6698     $restore_deactivate_volumes->($cfg, $devinfo);
6699
6700     close($fifofh) if $fifofh;
6701     unlink $mapfifo;
6702     rmtree $tmpdir;
6703
6704     if ($err) {
6705         $restore_destroy_volumes->($cfg, $devinfo);
6706         die $err;
6707     }
6708
6709     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6710
6711     PVE::Cluster::cfs_update(); # make sure we read new file
6712
6713     eval { rescan($vmid, 1); };
6714     warn $@ if $@;
6715
6716     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6717 }
6718
6719 sub restore_tar_archive {
6720     my ($archive, $vmid, $user, $opts) = @_;
6721
6722     if ($archive ne '-') {
6723         my $firstfile = tar_archive_read_firstfile($archive);
6724         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6725             if $firstfile ne 'qemu-server.conf';
6726     }
6727
6728     my $storecfg = PVE::Storage::config();
6729
6730     # avoid zombie disks when restoring over an existing VM -> cleanup first
6731     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6732     # skiplock=1 because qmrestore has set the 'create' lock itself already
6733     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6734     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6735
6736     my $tocmd = "/usr/lib/qemu-server/qmextract";
6737
6738     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6739     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6740     $tocmd .= ' --prealloc' if $opts->{prealloc};
6741     $tocmd .= ' --info' if $opts->{info};
6742
6743     # tar option "xf" does not autodetect compression when read from STDIN,
6744     # so we pipe to zcat
6745     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6746         PVE::Tools::shellquote("--to-command=$tocmd");
6747
6748     my $tmpdir = "/var/tmp/vzdumptmp$$";
6749     mkpath $tmpdir;
6750
6751     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6752     local $ENV{VZDUMP_VMID} = $vmid;
6753     local $ENV{VZDUMP_USER} = $user;
6754
6755     my $conffile = PVE::QemuConfig->config_file($vmid);
6756     my $new_conf_raw = '';
6757
6758     # disable interrupts (always do cleanups)
6759     local $SIG{INT} =
6760         local $SIG{TERM} =
6761         local $SIG{QUIT} =
6762         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6763
6764     eval {
6765         # enable interrupts
6766         local $SIG{INT} =
6767             local $SIG{TERM} =
6768             local $SIG{QUIT} =
6769             local $SIG{HUP} =
6770             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6771
6772         if ($archive eq '-') {
6773             print "extracting archive from STDIN\n";
6774             run_command($cmd, input => "<&STDIN");
6775         } else {
6776             print "extracting archive '$archive'\n";
6777             run_command($cmd);
6778         }
6779
6780         return if $opts->{info};
6781
6782         # read new mapping
6783         my $map = {};
6784         my $statfile = "$tmpdir/qmrestore.stat";
6785         if (my $fd = IO::File->new($statfile, "r")) {
6786             while (defined (my $line = <$fd>)) {
6787                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6788                     $map->{$1} = $2 if $1;
6789                 } else {
6790                     print STDERR "unable to parse line in statfile - $line\n";
6791                 }
6792             }
6793             $fd->close();
6794         }
6795
6796         my $confsrc = "$tmpdir/qemu-server.conf";
6797
6798         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6799
6800         my $cookie = { netcount => 0 };
6801         while (defined (my $line = <$srcfd>)) {
6802             $new_conf_raw .= restore_update_config_line(
6803                 $cookie,
6804                 $map,
6805                 $line,
6806                 $opts->{unique},
6807             );
6808         }
6809
6810         $srcfd->close();
6811     };
6812     if (my $err = $@) {
6813         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6814         die $err;
6815     }
6816
6817     rmtree $tmpdir;
6818
6819     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6820
6821     PVE::Cluster::cfs_update(); # make sure we read new file
6822
6823     eval { rescan($vmid, 1); };
6824     warn $@ if $@;
6825 };
6826
6827 sub foreach_storage_used_by_vm {
6828     my ($conf, $func) = @_;
6829
6830     my $sidhash = {};
6831
6832     PVE::QemuConfig->foreach_volume($conf, sub {
6833         my ($ds, $drive) = @_;
6834         return if drive_is_cdrom($drive);
6835
6836         my $volid = $drive->{file};
6837
6838         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6839         $sidhash->{$sid} = $sid if $sid;
6840     });
6841
6842     foreach my $sid (sort keys %$sidhash) {
6843         &$func($sid);
6844     }
6845 }
6846
6847 my $qemu_snap_storage = {
6848     rbd => 1,
6849 };
6850 sub do_snapshots_with_qemu {
6851     my ($storecfg, $volid) = @_;
6852
6853     my $storage_name = PVE::Storage::parse_volume_id($volid);
6854     my $scfg = $storecfg->{ids}->{$storage_name};
6855     die "could not find storage '$storage_name'\n" if !defined($scfg);
6856
6857     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6858         return 1;
6859     }
6860
6861     if ($volid =~ m/\.(qcow2|qed)$/){
6862         return 1;
6863     }
6864
6865     return;
6866 }
6867
6868 sub qga_check_running {
6869     my ($vmid, $nowarn) = @_;
6870
6871     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6872     if ($@) {
6873         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6874         return 0;
6875     }
6876     return 1;
6877 }
6878
6879 sub template_create {
6880     my ($vmid, $conf, $disk) = @_;
6881
6882     my $storecfg = PVE::Storage::config();
6883
6884     PVE::QemuConfig->foreach_volume($conf, sub {
6885         my ($ds, $drive) = @_;
6886
6887         return if drive_is_cdrom($drive);
6888         return if $disk && $ds ne $disk;
6889
6890         my $volid = $drive->{file};
6891         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6892
6893         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6894         $drive->{file} = $voliddst;
6895         $conf->{$ds} = print_drive($drive);
6896         PVE::QemuConfig->write_config($vmid, $conf);
6897     });
6898 }
6899
6900 sub convert_iscsi_path {
6901     my ($path) = @_;
6902
6903     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6904         my $portal = $1;
6905         my $target = $2;
6906         my $lun = $3;
6907
6908         my $initiator_name = get_initiator_name();
6909
6910         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6911                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6912     }
6913
6914     die "cannot convert iscsi path '$path', unkown format\n";
6915 }
6916
6917 sub qemu_img_convert {
6918     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6919
6920     my $storecfg = PVE::Storage::config();
6921     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6922     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6923
6924     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6925
6926     my $cachemode;
6927     my $src_path;
6928     my $src_is_iscsi = 0;
6929     my $src_format;
6930
6931     if ($src_storeid) {
6932         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6933         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6934         $src_format = qemu_img_format($src_scfg, $src_volname);
6935         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6936         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6937         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6938     } elsif (-f $src_volid) {
6939         $src_path = $src_volid;
6940         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6941             $src_format = $1;
6942         }
6943     }
6944
6945     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6946
6947     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6948     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6949     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6950     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6951
6952     my $cmd = [];
6953     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6954     push @$cmd, '-l', "snapshot.name=$snapname"
6955         if $snapname && $src_format && $src_format eq "qcow2";
6956     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6957     push @$cmd, '-T', $cachemode if defined($cachemode);
6958
6959     if ($src_is_iscsi) {
6960         push @$cmd, '--image-opts';
6961         $src_path = convert_iscsi_path($src_path);
6962     } elsif ($src_format) {
6963         push @$cmd, '-f', $src_format;
6964     }
6965
6966     if ($dst_is_iscsi) {
6967         push @$cmd, '--target-image-opts';
6968         $dst_path = convert_iscsi_path($dst_path);
6969     } else {
6970         push @$cmd, '-O', $dst_format;
6971     }
6972
6973     push @$cmd, $src_path;
6974
6975     if (!$dst_is_iscsi && $is_zero_initialized) {
6976         push @$cmd, "zeroinit:$dst_path";
6977     } else {
6978         push @$cmd, $dst_path;
6979     }
6980
6981     my $parser = sub {
6982         my $line = shift;
6983         if($line =~ m/\((\S+)\/100\%\)/){
6984             my $percent = $1;
6985             my $transferred = int($size * $percent / 100);
6986             my $total_h = render_bytes($size, 1);
6987             my $transferred_h = render_bytes($transferred, 1);
6988
6989             print "transferred $transferred_h of $total_h ($percent%)\n";
6990         }
6991
6992     };
6993
6994     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6995     my $err = $@;
6996     die "copy failed: $err" if $err;
6997 }
6998
6999 sub qemu_img_format {
7000     my ($scfg, $volname) = @_;
7001
7002     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7003         return $1;
7004     } else {
7005         return "raw";
7006     }
7007 }
7008
7009 sub qemu_drive_mirror {
7010     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7011
7012     $jobs = {} if !$jobs;
7013
7014     my $qemu_target;
7015     my $format;
7016     $jobs->{"drive-$drive"} = {};
7017
7018     if ($dst_volid =~ /^nbd:/) {
7019         $qemu_target = $dst_volid;
7020         $format = "nbd";
7021     } else {
7022         my $storecfg = PVE::Storage::config();
7023         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7024
7025         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7026
7027         $format = qemu_img_format($dst_scfg, $dst_volname);
7028
7029         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7030
7031         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7032     }
7033
7034     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7035     $opts->{format} = $format if $format;
7036
7037     if (defined($src_bitmap)) {
7038         $opts->{sync} = 'incremental';
7039         $opts->{bitmap} = $src_bitmap;
7040         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7041     }
7042
7043     if (defined($bwlimit)) {
7044         $opts->{speed} = $bwlimit * 1024;
7045         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7046     } else {
7047         print "drive mirror is starting for drive-$drive\n";
7048     }
7049
7050     # if a job already runs for this device we get an error, catch it for cleanup
7051     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7052     if (my $err = $@) {
7053         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7054         warn "$@\n" if $@;
7055         die "mirroring error: $err\n";
7056     }
7057
7058     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7059 }
7060
7061 # $completion can be either
7062 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7063 # 'cancel': wait until all jobs are ready, block-job-cancel them
7064 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7065 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7066 sub qemu_drive_mirror_monitor {
7067     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7068
7069     $completion //= 'complete';
7070     $op //= "mirror";
7071
7072     eval {
7073         my $err_complete = 0;
7074
7075         my $starttime = time ();
7076         while (1) {
7077             die "block job ('$op') timed out\n" if $err_complete > 300;
7078
7079             my $stats = mon_cmd($vmid, "query-block-jobs");
7080             my $ctime = time();
7081
7082             my $running_jobs = {};
7083             for my $stat (@$stats) {
7084                 next if $stat->{type} ne $op;
7085                 $running_jobs->{$stat->{device}} = $stat;
7086             }
7087
7088             my $readycounter = 0;
7089
7090             for my $job_id (sort keys %$jobs) {
7091                 my $job = $running_jobs->{$job_id};
7092
7093                 my $vanished = !defined($job);
7094                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7095                 if($complete || ($vanished && $completion eq 'auto')) {
7096                     print "$job_id: $op-job finished\n";
7097                     delete $jobs->{$job_id};
7098                     next;
7099                 }
7100
7101                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7102
7103                 my $busy = $job->{busy};
7104                 my $ready = $job->{ready};
7105                 if (my $total = $job->{len}) {
7106                     my $transferred = $job->{offset} || 0;
7107                     my $remaining = $total - $transferred;
7108                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7109
7110                     my $duration = $ctime - $starttime;
7111                     my $total_h = render_bytes($total, 1);
7112                     my $transferred_h = render_bytes($transferred, 1);
7113
7114                     my $status = sprintf(
7115                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7116                         render_duration($duration),
7117                     );
7118
7119                     if ($ready) {
7120                         if ($busy) {
7121                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7122                         } else {
7123                             $status .= ", ready";
7124                         }
7125                     }
7126                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7127                     $jobs->{$job_id}->{ready} = $ready;
7128                 }
7129
7130                 $readycounter++ if $job->{ready};
7131             }
7132
7133             last if scalar(keys %$jobs) == 0;
7134
7135             if ($readycounter == scalar(keys %$jobs)) {
7136                 print "all '$op' jobs are ready\n";
7137
7138                 # do the complete later (or has already been done)
7139                 last if $completion eq 'skip' || $completion eq 'auto';
7140
7141                 if ($vmiddst && $vmiddst != $vmid) {
7142                     my $agent_running = $qga && qga_check_running($vmid);
7143                     if ($agent_running) {
7144                         print "freeze filesystem\n";
7145                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7146                     } else {
7147                         print "suspend vm\n";
7148                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7149                     }
7150
7151                     # if we clone a disk for a new target vm, we don't switch the disk
7152                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7153
7154                     if ($agent_running) {
7155                         print "unfreeze filesystem\n";
7156                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7157                     } else {
7158                         print "resume vm\n";
7159                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7160                     }
7161
7162                     last;
7163                 } else {
7164
7165                     for my $job_id (sort keys %$jobs) {
7166                         # try to switch the disk if source and destination are on the same guest
7167                         print "$job_id: Completing block job_id...\n";
7168
7169                         my $op;
7170                         if ($completion eq 'complete') {
7171                             $op = 'block-job-complete';
7172                         } elsif ($completion eq 'cancel') {
7173                             $op = 'block-job-cancel';
7174                         } else {
7175                             die "invalid completion value: $completion\n";
7176                         }
7177                         eval { mon_cmd($vmid, $op, device => $job_id) };
7178                         if ($@ =~ m/cannot be completed/) {
7179                             print "$job_id: block job cannot be completed, trying again.\n";
7180                             $err_complete++;
7181                         }else {
7182                             print "$job_id: Completed successfully.\n";
7183                             $jobs->{$job_id}->{complete} = 1;
7184                         }
7185                     }
7186                 }
7187             }
7188             sleep 1;
7189         }
7190     };
7191     my $err = $@;
7192
7193     if ($err) {
7194         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7195         die "block job ($op) error: $err";
7196     }
7197 }
7198
7199 sub qemu_blockjobs_cancel {
7200     my ($vmid, $jobs) = @_;
7201
7202     foreach my $job (keys %$jobs) {
7203         print "$job: Cancelling block job\n";
7204         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7205         $jobs->{$job}->{cancel} = 1;
7206     }
7207
7208     while (1) {
7209         my $stats = mon_cmd($vmid, "query-block-jobs");
7210
7211         my $running_jobs = {};
7212         foreach my $stat (@$stats) {
7213             $running_jobs->{$stat->{device}} = $stat;
7214         }
7215
7216         foreach my $job (keys %$jobs) {
7217
7218             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7219                 print "$job: Done.\n";
7220                 delete $jobs->{$job};
7221             }
7222         }
7223
7224         last if scalar(keys %$jobs) == 0;
7225
7226         sleep 1;
7227     }
7228 }
7229
7230 sub clone_disk {
7231     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7232         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7233
7234     my $newvolid;
7235
7236     if (!$full) {
7237         print "create linked clone of drive $drivename ($drive->{file})\n";
7238         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7239         push @$newvollist, $newvolid;
7240     } else {
7241
7242         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7243         $storeid = $storage if $storage;
7244
7245         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7246
7247         print "create full clone of drive $drivename ($drive->{file})\n";
7248         my $name = undef;
7249         my $size = undef;
7250         if (drive_is_cloudinit($drive)) {
7251             $name = "vm-$newvmid-cloudinit";
7252             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7253             if ($scfg->{path}) {
7254                 $name .= ".$dst_format";
7255             }
7256             $snapname = undef;
7257             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7258         } elsif ($drivename eq 'efidisk0') {
7259             $size = get_efivars_size($conf);
7260         } else {
7261             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7262         }
7263         $newvolid = PVE::Storage::vdisk_alloc(
7264             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7265         );
7266         push @$newvollist, $newvolid;
7267
7268         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7269
7270         if (drive_is_cloudinit($drive)) {
7271             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7272             # if this is the case, we have to complete any block-jobs still there from
7273             # previous drive-mirrors
7274             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7275                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7276             }
7277             goto no_data_clone;
7278         }
7279
7280         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7281         if (!$running || $snapname) {
7282             # TODO: handle bwlimits
7283             if ($drivename eq 'efidisk0') {
7284                 # the relevant data on the efidisk may be smaller than the source
7285                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7286                 # that is given by the OVMF_VARS.fd
7287                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7288                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7289
7290                 # better for Ceph if block size is not too small, see bug #3324
7291                 my $bs = 1024*1024;
7292
7293                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7294                     "if=$src_path", "of=$dst_path"]);
7295             } else {
7296                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7297             }
7298         } else {
7299
7300             my $kvmver = get_running_qemu_version ($vmid);
7301             if (!min_version($kvmver, 2, 7)) {
7302                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7303                     if $drive->{iothread};
7304             }
7305
7306             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7307                 $completion, $qga, $bwlimit);
7308         }
7309     }
7310
7311 no_data_clone:
7312     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7313
7314     my $disk = $drive;
7315     $disk->{format} = undef;
7316     $disk->{file} = $newvolid;
7317     $disk->{size} = $size if defined($size);
7318
7319     return $disk;
7320 }
7321
7322 sub get_running_qemu_version {
7323     my ($vmid) = @_;
7324     my $res = mon_cmd($vmid, "query-version");
7325     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7326 }
7327
7328 sub qemu_use_old_bios_files {
7329     my ($machine_type) = @_;
7330
7331     return if !$machine_type;
7332
7333     my $use_old_bios_files = undef;
7334
7335     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7336         $machine_type = $1;
7337         $use_old_bios_files = 1;
7338     } else {
7339         my $version = extract_version($machine_type, kvm_user_version());
7340         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7341         # load new efi bios files on migration. So this hack is required to allow
7342         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7343         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7344         $use_old_bios_files = !min_version($version, 2, 4);
7345     }
7346
7347     return ($use_old_bios_files, $machine_type);
7348 }
7349
7350 sub get_efivars_size {
7351     my ($conf) = @_;
7352     my $arch = get_vm_arch($conf);
7353     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7354     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7355     return -s $ovmf_vars;
7356 }
7357
7358 sub update_efidisk_size {
7359     my ($conf) = @_;
7360
7361     return if !defined($conf->{efidisk0});
7362
7363     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7364     $disk->{size} = get_efivars_size($conf);
7365     $conf->{efidisk0} = print_drive($disk);
7366
7367     return;
7368 }
7369
7370 sub create_efidisk($$$$$) {
7371     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7372
7373     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7374     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7375
7376     my $vars_size_b = -s $ovmf_vars;
7377     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7378     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7379     PVE::Storage::activate_volumes($storecfg, [$volid]);
7380
7381     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7382     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7383
7384     return ($volid, $size/1024);
7385 }
7386
7387 sub vm_iothreads_list {
7388     my ($vmid) = @_;
7389
7390     my $res = mon_cmd($vmid, 'query-iothreads');
7391
7392     my $iothreads = {};
7393     foreach my $iothread (@$res) {
7394         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7395     }
7396
7397     return $iothreads;
7398 }
7399
7400 sub scsihw_infos {
7401     my ($conf, $drive) = @_;
7402
7403     my $maxdev = 0;
7404
7405     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7406         $maxdev = 7;
7407     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7408         $maxdev = 1;
7409     } else {
7410         $maxdev = 256;
7411     }
7412
7413     my $controller = int($drive->{index} / $maxdev);
7414     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7415         ? "virtioscsi"
7416         : "scsihw";
7417
7418     return ($maxdev, $controller, $controller_prefix);
7419 }
7420
7421 sub windows_version {
7422     my ($ostype) = @_;
7423
7424     return 0 if !$ostype;
7425
7426     my $winversion = 0;
7427
7428     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7429         $winversion = 5;
7430     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7431         $winversion = 6;
7432     } elsif ($ostype =~ m/^win(\d+)$/) {
7433         $winversion = $1;
7434     }
7435
7436     return $winversion;
7437 }
7438
7439 sub resolve_dst_disk_format {
7440         my ($storecfg, $storeid, $src_volname, $format) = @_;
7441         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7442
7443         if (!$format) {
7444             # if no target format is specified, use the source disk format as hint
7445             if ($src_volname) {
7446                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7447                 $format = qemu_img_format($scfg, $src_volname);
7448             } else {
7449                 return $defFormat;
7450             }
7451         }
7452
7453         # test if requested format is supported - else use default
7454         my $supported = grep { $_ eq $format } @$validFormats;
7455         $format = $defFormat if !$supported;
7456         return $format;
7457 }
7458
7459 # NOTE: if this logic changes, please update docs & possibly gui logic
7460 sub find_vmstate_storage {
7461     my ($conf, $storecfg) = @_;
7462
7463     # first, return storage from conf if set
7464     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7465
7466     my ($target, $shared, $local);
7467
7468     foreach_storage_used_by_vm($conf, sub {
7469         my ($sid) = @_;
7470         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7471         my $dst = $scfg->{shared} ? \$shared : \$local;
7472         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7473     });
7474
7475     # second, use shared storage where VM has at least one disk
7476     # third, use local storage where VM has at least one disk
7477     # fall back to local storage
7478     $target = $shared // $local // 'local';
7479
7480     return $target;
7481 }
7482
7483 sub generate_uuid {
7484     my ($uuid, $uuid_str);
7485     UUID::generate($uuid);
7486     UUID::unparse($uuid, $uuid_str);
7487     return $uuid_str;
7488 }
7489
7490 sub generate_smbios1_uuid {
7491     return "uuid=".generate_uuid();
7492 }
7493
7494 sub nbd_stop {
7495     my ($vmid) = @_;
7496
7497     mon_cmd($vmid, 'nbd-server-stop');
7498 }
7499
7500 sub create_reboot_request {
7501     my ($vmid) = @_;
7502     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7503         or die "failed to create reboot trigger file: $!\n";
7504     close($fh);
7505 }
7506
7507 sub clear_reboot_request {
7508     my ($vmid) = @_;
7509     my $path = "/run/qemu-server/$vmid.reboot";
7510     my $res = 0;
7511
7512     $res = unlink($path);
7513     die "could not remove reboot request for $vmid: $!"
7514         if !$res && $! != POSIX::ENOENT;
7515
7516     return $res;
7517 }
7518
7519 sub bootorder_from_legacy {
7520     my ($conf, $bootcfg) = @_;
7521
7522     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7523     my $bootindex_hash = {};
7524     my $i = 1;
7525     foreach my $o (split(//, $boot)) {
7526         $bootindex_hash->{$o} = $i*100;
7527         $i++;
7528     }
7529
7530     my $bootorder = {};
7531
7532     PVE::QemuConfig->foreach_volume($conf, sub {
7533         my ($ds, $drive) = @_;
7534
7535         if (drive_is_cdrom ($drive, 1)) {
7536             if ($bootindex_hash->{d}) {
7537                 $bootorder->{$ds} = $bootindex_hash->{d};
7538                 $bootindex_hash->{d} += 1;
7539             }
7540         } elsif ($bootindex_hash->{c}) {
7541             $bootorder->{$ds} = $bootindex_hash->{c}
7542                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7543             $bootindex_hash->{c} += 1;
7544         }
7545     });
7546
7547     if ($bootindex_hash->{n}) {
7548         for (my $i = 0; $i < $MAX_NETS; $i++) {
7549             my $netname = "net$i";
7550             next if !$conf->{$netname};
7551             $bootorder->{$netname} = $bootindex_hash->{n};
7552             $bootindex_hash->{n} += 1;
7553         }
7554     }
7555
7556     return $bootorder;
7557 }
7558
7559 # Generate default device list for 'boot: order=' property. Matches legacy
7560 # default boot order, but with explicit device names. This is important, since
7561 # the fallback for when neither 'order' nor the old format is specified relies
7562 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7563 sub get_default_bootdevices {
7564     my ($conf) = @_;
7565
7566     my @ret = ();
7567
7568     # harddisk
7569     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7570     push @ret, $first if $first;
7571
7572     # cdrom
7573     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7574     push @ret, $first if $first;
7575
7576     # network
7577     for (my $i = 0; $i < $MAX_NETS; $i++) {
7578         my $netname = "net$i";
7579         next if !$conf->{$netname};
7580         push @ret, $netname;
7581         last;
7582     }
7583
7584     return \@ret;
7585 }
7586
7587 sub device_bootorder {
7588     my ($conf) = @_;
7589
7590     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7591
7592     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7593
7594     my $bootorder = {};
7595     if (!defined($boot) || $boot->{legacy}) {
7596         $bootorder = bootorder_from_legacy($conf, $boot);
7597     } elsif ($boot->{order}) {
7598         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7599         for my $dev (PVE::Tools::split_list($boot->{order})) {
7600             $bootorder->{$dev} = $i++;
7601         }
7602     }
7603
7604     return $bootorder;
7605 }
7606
7607 sub register_qmeventd_handle {
7608     my ($vmid) = @_;
7609
7610     my $fh;
7611     my $peer = "/var/run/qmeventd.sock";
7612     my $count = 0;
7613
7614     for (;;) {
7615         $count++;
7616         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7617         last if $fh;
7618         if ($! != EINTR && $! != EAGAIN) {
7619             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7620         }
7621         if ($count > 4) {
7622             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7623               . "after $count retries\n";
7624         }
7625         usleep(25000);
7626     }
7627
7628     # send handshake to mark VM as backing up
7629     print $fh to_json({vzdump => {vmid => "$vmid"}});
7630
7631     # return handle to be closed later when inhibit is no longer required
7632     return $fh;
7633 }
7634
7635 # bash completion helper
7636
7637 sub complete_backup_archives {
7638     my ($cmdname, $pname, $cvalue) = @_;
7639
7640     my $cfg = PVE::Storage::config();
7641
7642     my $storeid;
7643
7644     if ($cvalue =~ m/^([^:]+):/) {
7645         $storeid = $1;
7646     }
7647
7648     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7649
7650     my $res = [];
7651     foreach my $id (keys %$data) {
7652         foreach my $item (@{$data->{$id}}) {
7653             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7654             push @$res, $item->{volid} if defined($item->{volid});
7655         }
7656     }
7657
7658     return $res;
7659 }
7660
7661 my $complete_vmid_full = sub {
7662     my ($running) = @_;
7663
7664     my $idlist = vmstatus();
7665
7666     my $res = [];
7667
7668     foreach my $id (keys %$idlist) {
7669         my $d = $idlist->{$id};
7670         if (defined($running)) {
7671             next if $d->{template};
7672             next if $running && $d->{status} ne 'running';
7673             next if !$running && $d->{status} eq 'running';
7674         }
7675         push @$res, $id;
7676
7677     }
7678     return $res;
7679 };
7680
7681 sub complete_vmid {
7682     return &$complete_vmid_full();
7683 }
7684
7685 sub complete_vmid_stopped {
7686     return &$complete_vmid_full(0);
7687 }
7688
7689 sub complete_vmid_running {
7690     return &$complete_vmid_full(1);
7691 }
7692
7693 sub complete_storage {
7694
7695     my $cfg = PVE::Storage::config();
7696     my $ids = $cfg->{ids};
7697
7698     my $res = [];
7699     foreach my $sid (keys %$ids) {
7700         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7701         next if !$ids->{$sid}->{content}->{images};
7702         push @$res, $sid;
7703     }
7704
7705     return $res;
7706 }
7707
7708 sub complete_migration_storage {
7709     my ($cmd, $param, $current_value, $all_args) = @_;
7710
7711     my $targetnode = @$all_args[1];
7712
7713     my $cfg = PVE::Storage::config();
7714     my $ids = $cfg->{ids};
7715
7716     my $res = [];
7717     foreach my $sid (keys %$ids) {
7718         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7719         next if !$ids->{$sid}->{content}->{images};
7720         push @$res, $sid;
7721     }
7722
7723     return $res;
7724 }
7725
7726 sub vm_is_paused {
7727     my ($vmid) = @_;
7728     my $qmpstatus = eval {
7729         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7730         mon_cmd($vmid, "query-status");
7731     };
7732     warn "$@\n" if $@;
7733     return $qmpstatus && $qmpstatus->{status} eq "paused";
7734 }
7735
7736 1;