PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => [
  67         "$EDK2_FW_BASE/OVMF_CODE.fd",
  68         "$EDK2_FW_BASE/OVMF_VARS.fd"
  69     ],
  70     aarch64 => [
  71         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  72         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  73     ],
  74 };
  75
  76 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  77
  78 # Note about locking: we use flock on the config file protect
  79 # against concurent actions.
  80 # Aditionaly, we have a 'lock' setting in the config file. This
  81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  82 # allowed when such lock is set. But you can ignore this kind of
  83 # lock with the --skiplock flag.
  84
  85 cfs_register_file('/qemu-server/',
  86                   \&parse_vm_config,
  87                   \&write_vm_config);
  88
  89 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  90     description => "Some command save/restore state from this location.",
  91     type => 'string',
  92     maxLength => 128,
  93     optional => 1,
  94 });
  95
  96 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  97         description => "Specifies the Qemu machine type.",
  98         type => 'string',
  99         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 100         maxLength => 40,
 101         optional => 1,
 102 });
 103
 104
 105 sub map_storage {
 106     my ($map, $source) = @_;
 107
 108     return $source if !defined($map);
 109
 110     return $map->{entries}->{$source}
 111         if $map->{entries} && defined($map->{entries}->{$source});
 112
 113     return $map->{default} if $map->{default};
 114
 115     # identity (fallback)
 116     return $source;
 117 }
 118
 119 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 120     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 121     type => 'string',
 122     format => 'storagepair-list',
 123     optional => 1,
 124 });
 125
 126 #no warnings 'redefine';
 127
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable Qemu GuestAgent.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     type => {
 166         description => "Select the agent type",
 167         type => 'string',
 168         default => 'virtio',
 169         optional => 1,
 170         enum => [qw(virtio isa)],
 171     },
 172 };
 173
 174 my $vga_fmt = {
 175     type => {
 176         description => "Select the VGA type.",
 177         type => 'string',
 178         default => 'std',
 179         optional => 1,
 180         default_key => 1,
 181         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 182     },
 183     memory => {
 184         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 185         type => 'integer',
 186         optional => 1,
 187         minimum => 4,
 188         maximum => 512,
 189     },
 190 };
 191
 192 my $ivshmem_fmt = {
 193     size => {
 194         type => 'integer',
 195         minimum => 1,
 196         description => "The size of the file in MB.",
 197     },
 198     name => {
 199         type => 'string',
 200         pattern => '[a-zA-Z0-9\-]+',
 201         optional => 1,
 202         format_description => 'string',
 203         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 204     },
 205 };
 206
 207 my $audio_fmt = {
 208     device => {
 209         type => 'string',
 210         enum => [qw(ich9-intel-hda intel-hda AC97)],
 211         description =>  "Configure an audio device."
 212     },
 213     driver =>  {
 214         type => 'string',
 215         enum => ['spice', 'none'],
 216         default => 'spice',
 217         optional => 1,
 218         description => "Driver backend for the audio device."
 219     },
 220 };
 221
 222 my $spice_enhancements_fmt = {
 223     foldersharing => {
 224         type => 'boolean',
 225         optional => 1,
 226         default => '0',
 227         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 228     },
 229     videostreaming =>  {
 230         type => 'string',
 231         enum => ['off', 'all', 'filter'],
 232         default => 'off',
 233         optional => 1,
 234         description => "Enable video streaming. Uses compression for detected video streams."
 235     },
 236 };
 237
 238 my $rng_fmt = {
 239     source => {
 240         type => 'string',
 241         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 242         default_key => 1,
 243         description => "The file on the host to gather entropy from. In most"
 244                      . " cases /dev/urandom should be preferred over /dev/random"
 245                      . " to avoid entropy-starvation issues on the host. Using"
 246                      . " urandom does *not* decrease security in any meaningful"
 247                      . " way, as it's still seeded from real entropy, and the"
 248                      . " bytes provided will most likely be mixed with real"
 249                      . " entropy on the guest as well. /dev/hwrng can be used"
 250                      . " to pass through a hardware RNG from the host.",
 251     },
 252     max_bytes => {
 253         type => 'integer',
 254         description => "Maximum bytes of entropy injected into the guest every"
 255                      . " 'period' milliseconds. Prefer a lower value when using"
 256                      . " /dev/random as source. Use 0 to disable limiting"
 257                      . " (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 261         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 262         # of overwhelming the host, provided we're reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota"
 268                      . " is reset, allowing the guest to retrieve another"
 269                      . " 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $confdesc = {
 276     onboot => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Specifies whether a VM will be started during system bootup.",
 280         default => 0,
 281     },
 282     autostart => {
 283         optional => 1,
 284         type => 'boolean',
 285         description => "Automatic restart after crash (currently ignored).",
 286         default => 0,
 287     },
 288     hotplug => {
 289         optional => 1,
 290         type => 'string', format => 'pve-hotplug-features',
 291         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 292         default => 'network,disk,usb',
 293     },
 294     reboot => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 298         default => 1,
 299     },
 300     lock => {
 301         optional => 1,
 302         type => 'string',
 303         description => "Lock/unlock the VM.",
 304         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 305     },
 306     cpulimit => {
 307         optional => 1,
 308         type => 'number',
 309         description => "Limit of CPU usage.",
 310         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 311         minimum => 0,
 312         maximum => 128,
 313         default => 0,
 314     },
 315     cpuunits => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "CPU weight for a VM.",
 319         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 320         minimum => 2,
 321         maximum => 262144,
 322         default => 1024,
 323     },
 324     memory => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 328         minimum => 16,
 329         default => 512,
 330     },
 331     balloon => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 335         minimum => 0,
 336     },
 337     shares => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 341         minimum => 0,
 342         maximum => 50000,
 343         default => 1000,
 344     },
 345     keyboard => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 349                        "It should not be necessary to set it.",
 350         enum => PVE::Tools::kvmkeymaplist(),
 351         default => undef,
 352     },
 353     name => {
 354         optional => 1,
 355         type => 'string', format => 'dns-name',
 356         description => "Set a name for the VM. Only used on the configuration web interface.",
 357     },
 358     scsihw => {
 359         optional => 1,
 360         type => 'string',
 361         description => "SCSI controller model",
 362         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 363         default => 'lsi',
 364     },
 365     description => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 369     },
 370     ostype => {
 371         optional => 1,
 372         type => 'string',
 373         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 374         description => "Specify guest operating system.",
 375         verbose_description => <<EODESC,
 376 Specify guest operating system. This is used to enable special
 377 optimization/features for specific operating systems:
 378
 379 [horizontal]
 380 other;; unspecified OS
 381 wxp;; Microsoft Windows XP
 382 w2k;; Microsoft Windows 2000
 383 w2k3;; Microsoft Windows 2003
 384 w2k8;; Microsoft Windows 2008
 385 wvista;; Microsoft Windows Vista
 386 win7;; Microsoft Windows 7
 387 win8;; Microsoft Windows 8/2012/2012r2
 388 win10;; Microsoft Windows 10/2016/2019
 389 l24;; Linux 2.4 Kernel
 390 l26;; Linux 2.6 - 5.X Kernel
 391 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 392 EODESC
 393     },
 394     boot => {
 395         optional => 1,
 396         type => 'string', format => 'pve-qm-boot',
 397         description => "Specify guest boot order. Use with 'order=', usage with"
 398                      . " no key or 'legacy=' is deprecated.",
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     keephugepages => {
 440         optional => 1,
 441         type => 'boolean',
 442         default => 0,
 443         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 444             ." after VM shutdown and can be used for subsequent starts.",
 445     },
 446     vcpus => {
 447         optional => 1,
 448         type => 'integer',
 449         description => "Number of hotplugged vcpus.",
 450         minimum => 1,
 451         default => 0,
 452     },
 453     acpi => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Enable/disable ACPI.",
 457         default => 1,
 458     },
 459     agent => {
 460         optional => 1,
 461         description => "Enable/disable Qemu GuestAgent and its properties.",
 462         type => 'string',
 463         format => $agent_fmt,
 464     },
 465     kvm => {
 466         optional => 1,
 467         type => 'boolean',
 468         description => "Enable/disable KVM hardware virtualization.",
 469         default => 1,
 470     },
 471     tdf => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable time drift fix.",
 475         default => 0,
 476     },
 477     localtime => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Set the real time clock to local time. This is enabled by default if ostype"
 481             ." indicates a Microsoft OS.",
 482     },
 483     freeze => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 487     },
 488     vga => {
 489         optional => 1,
 490         type => 'string', format => $vga_fmt,
 491         description => "Configure the VGA hardware.",
 492         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 493             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 494             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 495             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 496             ." display server. For win* OS you can select how many independent displays you want,"
 497             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 498             ." using a serial device as terminal.",
 499     },
 500     watchdog => {
 501         optional => 1,
 502         type => 'string', format => 'pve-qm-watchdog',
 503         description => "Create a virtual hardware watchdog device.",
 504         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 505             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 506             ." else the watchdog will reset the guest (or execute the respective action specified)",
 507     },
 508     startdate => {
 509         optional => 1,
 510         type => 'string',
 511         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 512         description => "Set the initial date of the real time clock. Valid format for date are:"
 513             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 514         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 515         default => 'now',
 516     },
 517     startup =>  get_standard_option('pve-startup-order'),
 518     template => {
 519         optional => 1,
 520         type => 'boolean',
 521         description => "Enable/disable Template.",
 522         default => 0,
 523     },
 524     args => {
 525         optional => 1,
 526         type => 'string',
 527         description => "Arbitrary arguments passed to kvm.",
 528         verbose_description => <<EODESCR,
 529 Arbitrary arguments passed to kvm, for example:
 530
 531 args: -no-reboot -no-hpet
 532
 533 NOTE: this option is for experts only.
 534 EODESCR
 535     },
 536     tablet => {
 537         optional => 1,
 538         type => 'boolean',
 539         default => 1,
 540         description => "Enable/disable the USB tablet device.",
 541         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 542             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 543             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 544             ." may consider disabling this to save some context switches. This is turned off by"
 545             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 546     },
 547     migrate_speed => {
 548         optional => 1,
 549         type => 'integer',
 550         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 551         minimum => 0,
 552         default => 0,
 553     },
 554     migrate_downtime => {
 555         optional => 1,
 556         type => 'number',
 557         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 558         minimum => 0,
 559         default => 0.1,
 560     },
 561     cdrom => {
 562         optional => 1,
 563         type => 'string', format => 'pve-qm-ide',
 564         typetext => '<volume>',
 565         description => "This is an alias for option -ide2",
 566     },
 567     cpu => {
 568         optional => 1,
 569         description => "Emulated CPU type.",
 570         type => 'string',
 571         format => 'pve-vm-cpu-conf',
 572     },
 573     parent => get_standard_option('pve-snapshot-name', {
 574         optional => 1,
 575         description => "Parent snapshot name. This is used internally, and should not be modified.",
 576     }),
 577     snaptime => {
 578         optional => 1,
 579         description => "Timestamp for snapshots.",
 580         type => 'integer',
 581         minimum => 0,
 582     },
 583     vmstate => {
 584         optional => 1,
 585         type => 'string', format => 'pve-volume-id',
 586         description => "Reference to a volume which stores the VM state. This is used internally"
 587             ." for snapshots.",
 588     },
 589     vmstatestorage => get_standard_option('pve-storage-id', {
 590         description => "Default storage for VM state volumes/files.",
 591         optional => 1,
 592     }),
 593     runningmachine => get_standard_option('pve-qemu-machine', {
 594         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 595             ." for snapshots.",
 596     }),
 597     runningcpu => {
 598         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 599             ." internally for snapshots.",
 600         optional => 1,
 601         type => 'string',
 602         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 603         format_description => 'QEMU -cpu parameter'
 604     },
 605     machine => get_standard_option('pve-qemu-machine'),
 606     arch => {
 607         description => "Virtual processor architecture. Defaults to the host.",
 608         optional => 1,
 609         type => 'string',
 610         enum => [qw(x86_64 aarch64)],
 611     },
 612     smbios1 => {
 613         description => "Specify SMBIOS type 1 fields.",
 614         type => 'string', format => 'pve-qm-smbios1',
 615         maxLength => 512,
 616         optional => 1,
 617     },
 618     protection => {
 619         optional => 1,
 620         type => 'boolean',
 621         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 622             ." remove disk operations.",
 623         default => 0,
 624     },
 625     bios => {
 626         optional => 1,
 627         type => 'string',
 628         enum => [ qw(seabios ovmf) ],
 629         description => "Select BIOS implementation.",
 630         default => 'seabios',
 631     },
 632     vmgenid => {
 633         type => 'string',
 634         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 635         format_description => 'UUID',
 636         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 637             ." to disable explicitly.",
 638         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 639             ." value identifier to the guest OS. This allows to notify the guest operating system"
 640             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 641             ." execution or creation from a template). The guest operating system notices the"
 642             ." change, and is then able to react as appropriate by marking its copies of"
 643             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 644             ."Note that auto-creation only works when done through API/CLI create or update methods"
 645             .", but not when manually editing the config file.",
 646         default => "1 (autogenerated)",
 647         optional => 1,
 648     },
 649     hookscript => {
 650         type => 'string',
 651         format => 'pve-volume-id',
 652         optional => 1,
 653         description => "Script that will be executed during various steps in the vms lifetime.",
 654     },
 655     ivshmem => {
 656         type => 'string',
 657         format => $ivshmem_fmt,
 658         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 659             ." the host.",
 660         optional => 1,
 661     },
 662     audio0 => {
 663         type => 'string',
 664         format => $audio_fmt,
 665         description => "Configure a audio device, useful in combination with QXL/Spice.",
 666         optional => 1
 667     },
 668     spice_enhancements => {
 669         type => 'string',
 670         format => $spice_enhancements_fmt,
 671         description => "Configure additional enhancements for SPICE.",
 672         optional => 1
 673     },
 674     tags => {
 675         type => 'string', format => 'pve-tag-list',
 676         description => 'Tags of the VM. This is only meta information.',
 677         optional => 1,
 678     },
 679     rng0 => {
 680         type => 'string',
 681         format => $rng_fmt,
 682         description => "Configure a VirtIO-based Random Number Generator.",
 683         optional => 1,
 684     },
 685 };
 686
 687 my $cicustom_fmt = {
 688     meta => {
 689         type => 'string',
 690         optional => 1,
 691         description => 'Specify a custom file containing all meta data passed to the VM via"
 692             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 693         format => 'pve-volume-id',
 694         format_description => 'volume',
 695     },
 696     network => {
 697         type => 'string',
 698         optional => 1,
 699         description => 'Specify a custom file containing all network data passed to the VM via'
 700             .' cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704     user => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all user data passed to the VM via'
 708             .' cloud-init.',
 709         format => 'pve-volume-id',
 710         format_description => 'volume',
 711     },
 712 };
 713 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 714
 715 my $confdesc_cloudinit = {
 716     citype => {
 717         optional => 1,
 718         type => 'string',
 719         description => 'Specifies the cloud-init configuration format. The default depends on the'
 720             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 721             .' and `configdrive2` for windows.',
 722         enum => ['configdrive2', 'nocloud', 'opennebula'],
 723     },
 724     ciuser => {
 725         optional => 1,
 726         type => 'string',
 727         description => "cloud-init: User name to change ssh keys and password for instead of the"
 728             ." image's configured default user.",
 729     },
 730     cipassword => {
 731         optional => 1,
 732         type => 'string',
 733         description => 'cloud-init: Password to assign the user. Using this is generally not'
 734             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 735             .' support hashed passwords.',
 736     },
 737     cicustom => {
 738         optional => 1,
 739         type => 'string',
 740         description => 'cloud-init: Specify custom files to replace the automatically generated'
 741             .' ones at start.',
 742         format => 'pve-qm-cicustom',
 743     },
 744     searchdomain => {
 745         optional => 1,
 746         type => 'string',
 747         description => "cloud-init: Sets DNS search domains for a container. Create will'
 748             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 749             .' are set.",
 750     },
 751     nameserver => {
 752         optional => 1,
 753         type => 'string', format => 'address-list',
 754         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 755             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 756             .' are set.",
 757     },
 758     sshkeys => {
 759         optional => 1,
 760         type => 'string',
 761         format => 'urlencoded',
 762         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 763     },
 764 };
 765
 766 # what about other qemu settings ?
 767 #cpu => 'string',
 768 #machine => 'string',
 769 #fda => 'file',
 770 #fdb => 'file',
 771 #mtdblock => 'file',
 772 #sd => 'file',
 773 #pflash => 'file',
 774 #snapshot => 'bool',
 775 #bootp => 'file',
 776 ##tftp => 'dir',
 777 ##smb => 'dir',
 778 #kernel => 'file',
 779 #append => 'string',
 780 #initrd => 'file',
 781 ##soundhw => 'string',
 782
 783 while (my ($k, $v) = each %$confdesc) {
 784     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 785 }
 786
 787 my $MAX_USB_DEVICES = 5;
 788 my $MAX_NETS = 32;
 789 my $MAX_SERIAL_PORTS = 4;
 790 my $MAX_PARALLEL_PORTS = 3;
 791 my $MAX_NUMA = 8;
 792
 793 my $numa_fmt = {
 794     cpus => {
 795         type => "string",
 796         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 797         description => "CPUs accessing this NUMA node.",
 798         format_description => "id[-id];...",
 799     },
 800     memory => {
 801         type => "number",
 802         description => "Amount of memory this NUMA node provides.",
 803         optional => 1,
 804     },
 805     hostnodes => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "Host NUMA nodes to use.",
 809         format_description => "id[-id];...",
 810         optional => 1,
 811     },
 812     policy => {
 813         type => 'string',
 814         enum => [qw(preferred bind interleave)],
 815         description => "NUMA allocation policy.",
 816         optional => 1,
 817     },
 818 };
 819 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 820 my $numadesc = {
 821     optional => 1,
 822     type => 'string', format => $numa_fmt,
 823     description => "NUMA topology.",
 824 };
 825 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 826
 827 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 828     $confdesc->{"numa$i"} = $numadesc;
 829 }
 830
 831 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 832                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 833                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 834 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 835
 836 my $net_fmt_bridge_descr = <<__EOD__;
 837 Bridge to attach the network device to. The Proxmox VE standard bridge
 838 is called 'vmbr0'.
 839
 840 If you do not specify a bridge, we create a kvm user (NATed) network
 841 device, which provides DHCP and DNS services. The following addresses
 842 are used:
 843
 844  10.0.2.2   Gateway
 845  10.0.2.3   DNS Server
 846  10.0.2.4   SMB Server
 847
 848 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 849 __EOD__
 850
 851 my $net_fmt = {
 852     macaddr  => get_standard_option('mac-addr', {
 853         description => "MAC address. That address must be unique withing your network. This is"
 854             ." automatically generated if not specified.",
 855     }),
 856     model => {
 857         type => 'string',
 858         description => "Network Card Model. The 'virtio' model provides the best performance with"
 859             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 860             ." best to use 'e1000'.",
 861         enum => $nic_model_list,
 862         default_key => 1,
 863     },
 864     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 865     bridge => {
 866         type => 'string',
 867         description => $net_fmt_bridge_descr,
 868         format_description => 'bridge',
 869         pattern => '[-_.\w\d]+',
 870         optional => 1,
 871     },
 872     queues => {
 873         type => 'integer',
 874         minimum => 0, maximum => 16,
 875         description => 'Number of packet queues to be used on the device.',
 876         optional => 1,
 877     },
 878     rate => {
 879         type => 'number',
 880         minimum => 0,
 881         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 882         optional => 1,
 883     },
 884     tag => {
 885         type => 'integer',
 886         minimum => 1, maximum => 4094,
 887         description => 'VLAN tag to apply to packets on this interface.',
 888         optional => 1,
 889     },
 890     trunks => {
 891         type => 'string',
 892         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 893         description => 'VLAN trunks to pass through this interface.',
 894         format_description => 'vlanid[;vlanid...]',
 895         optional => 1,
 896     },
 897     firewall => {
 898         type => 'boolean',
 899         description => 'Whether this interface should be protected by the firewall.',
 900         optional => 1,
 901     },
 902     link_down => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be disconnected (like pulling the plug).',
 905         optional => 1,
 906     },
 907     mtu => {
 908         type => 'integer',
 909         minimum => 1, maximum => 65520,
 910         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 911         optional => 1,
 912     },
 913 };
 914
 915 my $netdesc = {
 916     optional => 1,
 917     type => 'string', format => $net_fmt,
 918     description => "Specify network devices.",
 919 };
 920
 921 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 922
 923 my $ipconfig_fmt = {
 924     ip => {
 925         type => 'string',
 926         format => 'pve-ipv4-config',
 927         format_description => 'IPv4Format/CIDR',
 928         description => 'IPv4 address in CIDR format.',
 929         optional => 1,
 930         default => 'dhcp',
 931     },
 932     gw => {
 933         type => 'string',
 934         format => 'ipv4',
 935         format_description => 'GatewayIPv4',
 936         description => 'Default gateway for IPv4 traffic.',
 937         optional => 1,
 938         requires => 'ip',
 939     },
 940     ip6 => {
 941         type => 'string',
 942         format => 'pve-ipv6-config',
 943         format_description => 'IPv6Format/CIDR',
 944         description => 'IPv6 address in CIDR format.',
 945         optional => 1,
 946         default => 'dhcp',
 947     },
 948     gw6 => {
 949         type => 'string',
 950         format => 'ipv6',
 951         format_description => 'GatewayIPv6',
 952         description => 'Default gateway for IPv6 traffic.',
 953         optional => 1,
 954         requires => 'ip6',
 955     },
 956 };
 957 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 958 my $ipconfigdesc = {
 959     optional => 1,
 960     type => 'string', format => 'pve-qm-ipconfig',
 961     description => <<'EODESCR',
 962 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 963
 964 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 965
 966 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 967 gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 969 cloud-init 19.4 or newer.
 970
 971 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 972 dhcp on IPv4.
 973 EODESCR
 974 };
 975 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 976
 977 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 978     $confdesc->{"net$i"} = $netdesc;
 979     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 980 }
 981
 982 foreach my $key (keys %$confdesc_cloudinit) {
 983     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 984 }
 985
 986 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 987 sub verify_volume_id_or_qm_path {
 988     my ($volid, $noerr) = @_;
 989
 990     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 991         return $volid;
 992     }
 993
 994     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 995     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 996     if ($@) {
 997         return if $noerr;
 998         die $@;
 999     }
1000     return $volid;
1001 }
1002
1003 my $usb_fmt = {
1004     host => {
1005         default_key => 1,
1006         type => 'string', format => 'pve-qm-usb-device',
1007         format_description => 'HOSTUSBDEVICE|spice',
1008         description => <<EODESCR,
1009 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1010
1011  'bus-port(.port)*' (decimal numbers) or
1012  'vendor_id:product_id' (hexadeciaml numbers) or
1013  'spice'
1014
1015 You can use the 'lsusb -t' command to list existing usb devices.
1016
1017 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1018 machines - use with special care.
1019
1020 The value 'spice' can be used to add a usb redirection devices for spice.
1021 EODESCR
1022     },
1023     usb3 => {
1024         optional => 1,
1025         type => 'boolean',
1026         description => "Specifies whether if given host option is a USB3 device or port.",
1027         default => 0,
1028     },
1029 };
1030
1031 my $usbdesc = {
1032     optional => 1,
1033     type => 'string', format => $usb_fmt,
1034     description => "Configure an USB device (n is 0 to 4).",
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1037
1038 my $serialdesc = {
1039         optional => 1,
1040         type => 'string',
1041         pattern => '(/dev/.+|socket)',
1042         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1043         verbose_description =>  <<EODESCR,
1044 Create a serial device inside the VM (n is 0 to 3), and pass through a
1045 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1046 host side (use 'qm terminal' to open a terminal connection).
1047
1048 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1049 use with special care.
1050
1051 CAUTION: Experimental! User reported problems with this option.
1052 EODESCR
1053 };
1054
1055 my $paralleldesc= {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1059         description =>  "Map host parallel devices (n is 0 to 2).",
1060         verbose_description =>  <<EODESCR,
1061 Map host parallel devices (n is 0 to 2).
1062
1063 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1064 machines - use with special care.
1065
1066 CAUTION: Experimental! User reported problems with this option.
1067 EODESCR
1068 };
1069
1070 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1071     $confdesc->{"parallel$i"} = $paralleldesc;
1072 }
1073
1074 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1075     $confdesc->{"serial$i"} = $serialdesc;
1076 }
1077
1078 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1079     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1080 }
1081
1082 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1083     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1084 }
1085
1086 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1087     $confdesc->{"usb$i"} = $usbdesc;
1088 }
1089
1090 my $boot_fmt = {
1091     legacy => {
1092         optional => 1,
1093         default_key => 1,
1094         type => 'string',
1095         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1096                      . " Deprecated, use 'order=' instead.",
1097         pattern => '[acdn]{1,4}',
1098         format_description => "[acdn]{1,4}",
1099
1100         # note: this is also the fallback if boot: is not given at all
1101         default => 'cdn',
1102     },
1103     order => {
1104         optional => 1,
1105         type => 'string',
1106         format => 'pve-qm-bootdev-list',
1107         format_description => "device[;device...]",
1108         description => <<EODESC,
1109 The guest will attempt to boot from devices in the order they appear here.
1110
1111 Disks, optical drives and passed-through storage USB devices will be directly
1112 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1113 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1114
1115 Note that only devices in this list will be marked as bootable and thus loaded
1116 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1117 (e.g. software-raid), you need to specify all of them here.
1118
1119 Overrides the deprecated 'legacy=[acdn]*' value when given.
1120 EODESC
1121     },
1122 };
1123 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1124
1125 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1126 sub verify_bootdev {
1127     my ($dev, $noerr) = @_;
1128
1129     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1130
1131     my $check = sub {
1132         my ($base) = @_;
1133         return 0 if $dev !~ m/^$base\d+$/;
1134         return 0 if !$confdesc->{$dev};
1135         return 1;
1136     };
1137
1138     return $dev if $check->("net");
1139     return $dev if $check->("usb");
1140     return $dev if $check->("hostpci");
1141
1142     return if $noerr;
1143     die "invalid boot device '$dev'\n";
1144 }
1145
1146 sub print_bootorder {
1147     my ($devs) = @_;
1148     return "" if !@$devs;
1149     my $data = { order => join(';', @$devs) };
1150     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1151 }
1152
1153 my $kvm_api_version = 0;
1154
1155 sub kvm_version {
1156     return $kvm_api_version if $kvm_api_version;
1157
1158     open my $fh, '<', '/dev/kvm' or return;
1159
1160     # 0xae00 => KVM_GET_API_VERSION
1161     $kvm_api_version = ioctl($fh, 0xae00, 0);
1162     close($fh);
1163
1164     return $kvm_api_version;
1165 }
1166
1167 my $kvm_user_version = {};
1168 my $kvm_mtime = {};
1169
1170 sub kvm_user_version {
1171     my ($binary) = @_;
1172
1173     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1174     my $st = stat($binary);
1175
1176     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1177     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1178         $cachedmtime == $st->mtime;
1179
1180     $kvm_user_version->{$binary} = 'unknown';
1181     $kvm_mtime->{$binary} = $st->mtime;
1182
1183     my $code = sub {
1184         my $line = shift;
1185         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1186             $kvm_user_version->{$binary} = $2;
1187         }
1188     };
1189
1190     eval { run_command([$binary, '--version'], outfunc => $code); };
1191     warn $@ if $@;
1192
1193     return $kvm_user_version->{$binary};
1194
1195 }
1196 my sub extract_version {
1197     my ($machine_type, $version) = @_;
1198     $version = kvm_user_version() if !defined($version);
1199     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1200 }
1201
1202 sub kernel_has_vhost_net {
1203     return -c '/dev/vhost-net';
1204 }
1205
1206 sub option_exists {
1207     my $key = shift;
1208     return defined($confdesc->{$key});
1209 }
1210
1211 my $cdrom_path;
1212 sub get_cdrom_path {
1213
1214     return  $cdrom_path if $cdrom_path;
1215
1216     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1217     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1218     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1219 }
1220
1221 sub get_iso_path {
1222     my ($storecfg, $vmid, $cdrom) = @_;
1223
1224     if ($cdrom eq 'cdrom') {
1225         return get_cdrom_path();
1226     } elsif ($cdrom eq 'none') {
1227         return '';
1228     } elsif ($cdrom =~ m|^/|) {
1229         return $cdrom;
1230     } else {
1231         return PVE::Storage::path($storecfg, $cdrom);
1232     }
1233 }
1234
1235 # try to convert old style file names to volume IDs
1236 sub filename_to_volume_id {
1237     my ($vmid, $file, $media) = @_;
1238
1239      if (!($file eq 'none' || $file eq 'cdrom' ||
1240           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1241
1242         return if $file =~ m|/|;
1243
1244         if ($media && $media eq 'cdrom') {
1245             $file = "local:iso/$file";
1246         } else {
1247             $file = "local:$vmid/$file";
1248         }
1249     }
1250
1251     return $file;
1252 }
1253
1254 sub verify_media_type {
1255     my ($opt, $vtype, $media) = @_;
1256
1257     return if !$media;
1258
1259     my $etype;
1260     if ($media eq 'disk') {
1261         $etype = 'images';
1262     } elsif ($media eq 'cdrom') {
1263         $etype = 'iso';
1264     } else {
1265         die "internal error";
1266     }
1267
1268     return if ($vtype eq $etype);
1269
1270     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1271 }
1272
1273 sub cleanup_drive_path {
1274     my ($opt, $storecfg, $drive) = @_;
1275
1276     # try to convert filesystem paths to volume IDs
1277
1278     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1279         ($drive->{file} !~ m|^/dev/.+|) &&
1280         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1281         ($drive->{file} !~ m/^\d+$/)) {
1282         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1283         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1284             if !$vtype;
1285         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1286         verify_media_type($opt, $vtype, $drive->{media});
1287         $drive->{file} = $volid;
1288     }
1289
1290     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1291 }
1292
1293 sub parse_hotplug_features {
1294     my ($data) = @_;
1295
1296     my $res = {};
1297
1298     return $res if $data eq '0';
1299
1300     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1301
1302     foreach my $feature (PVE::Tools::split_list($data)) {
1303         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304             $res->{$1} = 1;
1305         } else {
1306             die "invalid hotplug feature '$feature'\n";
1307         }
1308     }
1309     return $res;
1310 }
1311
1312 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1313 sub pve_verify_hotplug_features {
1314     my ($value, $noerr) = @_;
1315
1316     return $value if parse_hotplug_features($value);
1317
1318     return if $noerr;
1319
1320     die "unable to parse hotplug option\n";
1321 }
1322
1323 sub scsi_inquiry {
1324     my($fh, $noerr) = @_;
1325
1326     my $SG_IO = 0x2285;
1327     my $SG_GET_VERSION_NUM = 0x2282;
1328
1329     my $versionbuf = "\x00" x 8;
1330     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1331     if (!$ret) {
1332         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1333         return;
1334     }
1335     my $version = unpack("I", $versionbuf);
1336     if ($version < 30000) {
1337         die "scsi generic interface too old\n"  if !$noerr;
1338         return;
1339     }
1340
1341     my $buf = "\x00" x 36;
1342     my $sensebuf = "\x00" x 8;
1343     my $cmd = pack("C x3 C x1", 0x12, 36);
1344
1345     # see /usr/include/scsi/sg.h
1346     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1347
1348     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1349                       length($sensebuf), 0, length($buf), $buf,
1350                       $cmd, $sensebuf, 6000);
1351
1352     $ret = ioctl($fh, $SG_IO, $packet);
1353     if (!$ret) {
1354         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1355         return;
1356     }
1357
1358     my @res = unpack($sg_io_hdr_t, $packet);
1359     if ($res[17] || $res[18]) {
1360         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1361         return;
1362     }
1363
1364     my $res = {};
1365     (my $byte0, my $byte1, $res->{vendor},
1366      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1367
1368     $res->{removable} = $byte1 & 128 ? 1 : 0;
1369     $res->{type} = $byte0 & 31;
1370
1371     return $res;
1372 }
1373
1374 sub path_is_scsi {
1375     my ($path) = @_;
1376
1377     my $fh = IO::File->new("+<$path") || return;
1378     my $res = scsi_inquiry($fh, 1);
1379     close($fh);
1380
1381     return $res;
1382 }
1383
1384 sub print_tabletdevice_full {
1385     my ($conf, $arch) = @_;
1386
1387     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1388
1389     # we use uhci for old VMs because tablet driver was buggy in older qemu
1390     my $usbbus;
1391     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1392         $usbbus = 'ehci';
1393     } else {
1394         $usbbus = 'uhci';
1395     }
1396
1397     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1398 }
1399
1400 sub print_keyboarddevice_full {
1401     my ($conf, $arch, $machine) = @_;
1402
1403     return if $arch ne 'aarch64';
1404
1405     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1406 }
1407
1408 my sub get_drive_id {
1409     my ($drive) = @_;
1410     return "$drive->{interface}$drive->{index}";
1411 }
1412
1413 sub print_drivedevice_full {
1414     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1415
1416     my $device = '';
1417     my $maxdev = 0;
1418
1419     my $drive_id = get_drive_id($drive);
1420     if ($drive->{interface} eq 'virtio') {
1421         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1422         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1423         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1424     } elsif ($drive->{interface} eq 'scsi') {
1425
1426         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1427         my $unit = $drive->{index} % $maxdev;
1428         my $devicetype = 'hd';
1429         my $path = '';
1430         if (drive_is_cdrom($drive)) {
1431             $devicetype = 'cd';
1432         } else {
1433             if ($drive->{file} =~ m|^/|) {
1434                 $path = $drive->{file};
1435                 if (my $info = path_is_scsi($path)) {
1436                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1437                         $devicetype = 'block';
1438                     } elsif ($info->{type} == 1) { # tape
1439                         $devicetype = 'generic';
1440                     }
1441                 }
1442             } else {
1443                  $path = PVE::Storage::path($storecfg, $drive->{file});
1444             }
1445
1446             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1447             my $version = extract_version($machine_type, kvm_user_version());
1448             if ($path =~ m/^iscsi\:\/\// &&
1449                !min_version($version, 4, 1)) {
1450                 $devicetype = 'generic';
1451             }
1452         }
1453
1454         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1455             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1456         } else {
1457             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1458                 .",lun=$drive->{index}";
1459         }
1460         $device .= ",drive=drive-$drive_id,id=$drive_id";
1461
1462         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1463             $device .= ",rotation_rate=1";
1464         }
1465         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1466
1467     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1468         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1469         my $controller = int($drive->{index} / $maxdev);
1470         my $unit = $drive->{index} % $maxdev;
1471         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1472
1473         $device = "ide-$devicetype";
1474         if ($drive->{interface} eq 'ide') {
1475             $device .= ",bus=ide.$controller,unit=$unit";
1476         } else {
1477             $device .= ",bus=ahci$controller.$unit";
1478         }
1479         $device .= ",drive=drive-$drive_id,id=$drive_id";
1480
1481         if ($devicetype eq 'hd') {
1482             if (my $model = $drive->{model}) {
1483                 $model = URI::Escape::uri_unescape($model);
1484                 $device .= ",model=$model";
1485             }
1486             if ($drive->{ssd}) {
1487                 $device .= ",rotation_rate=1";
1488             }
1489         }
1490         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1491     } elsif ($drive->{interface} eq 'usb') {
1492         die "implement me";
1493         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1494     } else {
1495         die "unsupported interface type";
1496     }
1497
1498     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1499
1500     if (my $serial = $drive->{serial}) {
1501         $serial = URI::Escape::uri_unescape($serial);
1502         $device .= ",serial=$serial";
1503     }
1504
1505
1506     return $device;
1507 }
1508
1509 sub get_initiator_name {
1510     my $initiator;
1511
1512     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1513     while (defined(my $line = <$fh>)) {
1514         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1515         $initiator = $1;
1516         last;
1517     }
1518     $fh->close();
1519
1520     return $initiator;
1521 }
1522
1523 sub print_drive_commandline_full {
1524     my ($storecfg, $vmid, $drive, $pbs_name) = @_;
1525
1526     my $path;
1527     my $volid = $drive->{file};
1528     my $format = $drive->{format};
1529     my $drive_id = get_drive_id($drive);
1530
1531     if (drive_is_cdrom($drive)) {
1532         $path = get_iso_path($storecfg, $vmid, $volid);
1533         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1534     } else {
1535         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1536         if ($storeid) {
1537             $path = PVE::Storage::path($storecfg, $volid);
1538             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1539             $format //= qemu_img_format($scfg, $volname);
1540         } else {
1541             $path = $volid;
1542             $format //= "raw";
1543         }
1544    }
1545
1546    my $is_rbd = $path =~ m/^rbd:/;
1547
1548     my $opts = '';
1549     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1550     foreach my $o (@qemu_drive_options) {
1551         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1552     }
1553
1554     # snapshot only accepts on|off
1555     if (defined($drive->{snapshot})) {
1556         my $v = $drive->{snapshot} ? 'on' : 'off';
1557         $opts .= ",snapshot=$v";
1558     }
1559
1560     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1561         my ($dir, $qmpname) = @$type;
1562         if (my $v = $drive->{"mbps$dir"}) {
1563             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1564         }
1565         if (my $v = $drive->{"mbps${dir}_max"}) {
1566             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1567         }
1568         if (my $v = $drive->{"bps${dir}_max_length"}) {
1569             $opts .= ",throttling.bps$qmpname-max-length=$v";
1570         }
1571         if (my $v = $drive->{"iops${dir}"}) {
1572             $opts .= ",throttling.iops$qmpname=$v";
1573         }
1574         if (my $v = $drive->{"iops${dir}_max"}) {
1575             $opts .= ",throttling.iops$qmpname-max=$v";
1576         }
1577         if (my $v = $drive->{"iops${dir}_max_length"}) {
1578             $opts .= ",throttling.iops$qmpname-max-length=$v";
1579         }
1580     }
1581
1582     if ($pbs_name) {
1583         $format = "rbd" if $is_rbd;
1584         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1585             if !$format;
1586         $opts .= ",format=alloc-track,file.driver=$format";
1587     } elsif ($format) {
1588         $opts .= ",format=$format";
1589     }
1590
1591     my $cache_direct = 0;
1592
1593     if (my $cache = $drive->{cache}) {
1594         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1595     } elsif (!drive_is_cdrom($drive)) {
1596         $opts .= ",cache=none";
1597         $cache_direct = 1;
1598     }
1599
1600     # aio native works only with O_DIRECT
1601     if (!$drive->{aio}) {
1602         if($cache_direct) {
1603             $opts .= ",aio=native";
1604         } else {
1605             $opts .= ",aio=threads";
1606         }
1607     }
1608
1609     if (!drive_is_cdrom($drive)) {
1610         my $detectzeroes;
1611         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1612             $detectzeroes = 'off';
1613         } elsif ($drive->{discard}) {
1614             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1615         } else {
1616             # This used to be our default with discard not being specified:
1617             $detectzeroes = 'on';
1618         }
1619
1620         # note: 'detect-zeroes' works per blockdev and we want it to persist
1621         # after the alloc-track is removed, so put it on 'file' directly
1622         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1623         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1624     }
1625
1626     if ($pbs_name) {
1627         $opts .= ",backing=$pbs_name";
1628         $opts .= ",auto-remove=on";
1629     }
1630
1631     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1632     my $file_param = "file";
1633     if ($pbs_name) {
1634         # non-rbd drivers require the underlying file to be a seperate block
1635         # node, so add a second .file indirection
1636         $file_param .= ".file" if !$is_rbd;
1637         $file_param .= ".filename";
1638     }
1639     my $pathinfo = $path ? "$file_param=$path," : '';
1640
1641     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1642 }
1643
1644 sub print_pbs_blockdev {
1645     my ($pbs_conf, $pbs_name) = @_;
1646     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1647     $blockdev .= ",repository=$pbs_conf->{repository}";
1648     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1649     $blockdev .= ",archive=$pbs_conf->{archive}";
1650     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1651     return $blockdev;
1652 }
1653
1654 sub print_netdevice_full {
1655     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1656
1657     my $device = $net->{model};
1658     if ($net->{model} eq 'virtio') {
1659          $device = 'virtio-net-pci';
1660      };
1661
1662     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1663     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1664     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1665         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1666         # and out of each queue plus one config interrupt and control vector queue
1667         my $vectors = $net->{queues} * 2 + 2;
1668         $tmpstr .= ",vectors=$vectors,mq=on";
1669     }
1670     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1671
1672     if (my $mtu = $net->{mtu}) {
1673         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1674             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1675             if ($mtu == 1) {
1676                  $mtu = $bridge_mtu;
1677             } elsif ($mtu < 576) {
1678                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1679             } elsif ($mtu > $bridge_mtu) {
1680                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1681             }
1682             $tmpstr .= ",host_mtu=$mtu";
1683         } else {
1684             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1685         }
1686     }
1687
1688     if ($use_old_bios_files) {
1689         my $romfile;
1690         if ($device eq 'virtio-net-pci') {
1691             $romfile = 'pxe-virtio.rom';
1692         } elsif ($device eq 'e1000') {
1693             $romfile = 'pxe-e1000.rom';
1694         } elsif ($device eq 'ne2k') {
1695             $romfile = 'pxe-ne2k_pci.rom';
1696         } elsif ($device eq 'pcnet') {
1697             $romfile = 'pxe-pcnet.rom';
1698         } elsif ($device eq 'rtl8139') {
1699             $romfile = 'pxe-rtl8139.rom';
1700         }
1701         $tmpstr .= ",romfile=$romfile" if $romfile;
1702     }
1703
1704     return $tmpstr;
1705 }
1706
1707 sub print_netdev_full {
1708     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1709
1710     my $i = '';
1711     if ($netid =~ m/^net(\d+)$/) {
1712         $i = int($1);
1713     }
1714
1715     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1716
1717     my $ifname = "tap${vmid}i$i";
1718
1719     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720     die "interface name '$ifname' is too long (max 15 character)\n"
1721         if length($ifname) >= 16;
1722
1723     my $vhostparam = '';
1724     if (is_native($arch)) {
1725         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1726     }
1727
1728     my $vmname = $conf->{name} || "vm$vmid";
1729
1730     my $netdev = "";
1731     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1732
1733     if ($net->{bridge}) {
1734         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1735             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1736     } else {
1737         $netdev = "type=user,id=$netid,hostname=$vmname";
1738     }
1739
1740     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1741
1742     return $netdev;
1743 }
1744
1745 my $vga_map = {
1746     'cirrus' => 'cirrus-vga',
1747     'std' => 'VGA',
1748     'vmware' => 'vmware-svga',
1749     'virtio' => 'virtio-vga',
1750 };
1751
1752 sub print_vga_device {
1753     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1754
1755     my $type = $vga_map->{$vga->{type}};
1756     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1757         $type = 'virtio-gpu';
1758     }
1759     my $vgamem_mb = $vga->{memory};
1760
1761     my $max_outputs = '';
1762     if ($qxlnum) {
1763         $type = $id ? 'qxl' : 'qxl-vga';
1764
1765         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1766             # set max outputs so linux can have up to 4 qxl displays with one device
1767             if (min_version($machine_version, 4, 1)) {
1768                 $max_outputs = ",max_outputs=4";
1769             }
1770         }
1771     }
1772
1773     die "no devicetype for $vga->{type}\n" if !$type;
1774
1775     my $memory = "";
1776     if ($vgamem_mb) {
1777         if ($vga->{type} eq 'virtio') {
1778             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1779             $memory = ",max_hostmem=$bytes";
1780         } elsif ($qxlnum) {
1781             # from https://www.spice-space.org/multiple-monitors.html
1782             $memory = ",vgamem_mb=$vga->{memory}";
1783             my $ram = $vgamem_mb * 4;
1784             my $vram = $vgamem_mb * 2;
1785             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1786         } else {
1787             $memory = ",vgamem_mb=$vga->{memory}";
1788         }
1789     } elsif ($qxlnum && $id) {
1790         $memory = ",ram_size=67108864,vram_size=33554432";
1791     }
1792
1793     my $edidoff = "";
1794     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1795         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1796     }
1797
1798     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1799     my $vgaid = "vga" . ($id // '');
1800     my $pciaddr;
1801
1802     if ($q35 && $vgaid eq 'vga') {
1803         # the first display uses pcie.0 bus on q35 machines
1804         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1805     } else {
1806         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1807     }
1808
1809     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1810 }
1811
1812 sub parse_number_sets {
1813     my ($set) = @_;
1814     my $res = [];
1815     foreach my $part (split(/;/, $set)) {
1816         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1817             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1818             push @$res, [ $1, $2 ];
1819         } else {
1820             die "invalid range: $part\n";
1821         }
1822     }
1823     return $res;
1824 }
1825
1826 sub parse_numa {
1827     my ($data) = @_;
1828
1829     my $res = parse_property_string($numa_fmt, $data);
1830     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1831     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1832     return $res;
1833 }
1834
1835 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1836 sub parse_net {
1837     my ($data) = @_;
1838
1839     my $res = eval { parse_property_string($net_fmt, $data) };
1840     if ($@) {
1841         warn $@;
1842         return;
1843     }
1844     if (!defined($res->{macaddr})) {
1845         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1846         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1847     }
1848     return $res;
1849 }
1850
1851 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1852 sub parse_ipconfig {
1853     my ($data) = @_;
1854
1855     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1856     if ($@) {
1857         warn $@;
1858         return;
1859     }
1860
1861     if ($res->{gw} && !$res->{ip}) {
1862         warn 'gateway specified without specifying an IP address';
1863         return;
1864     }
1865     if ($res->{gw6} && !$res->{ip6}) {
1866         warn 'IPv6 gateway specified without specifying an IPv6 address';
1867         return;
1868     }
1869     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1870         warn 'gateway specified together with DHCP';
1871         return;
1872     }
1873     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1874         # gw6 + auto/dhcp
1875         warn "IPv6 gateway specified together with $res->{ip6} address";
1876         return;
1877     }
1878
1879     if (!$res->{ip} && !$res->{ip6}) {
1880         return { ip => 'dhcp', ip6 => 'dhcp' };
1881     }
1882
1883     return $res;
1884 }
1885
1886 sub print_net {
1887     my $net = shift;
1888
1889     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1890 }
1891
1892 sub add_random_macs {
1893     my ($settings) = @_;
1894
1895     foreach my $opt (keys %$settings) {
1896         next if $opt !~ m/^net(\d+)$/;
1897         my $net = parse_net($settings->{$opt});
1898         next if !$net;
1899         $settings->{$opt} = print_net($net);
1900     }
1901 }
1902
1903 sub vm_is_volid_owner {
1904     my ($storecfg, $vmid, $volid) = @_;
1905
1906     if ($volid !~  m|^/|) {
1907         my ($path, $owner);
1908         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1909         if ($owner && ($owner == $vmid)) {
1910             return 1;
1911         }
1912     }
1913
1914     return;
1915 }
1916
1917 sub vmconfig_register_unused_drive {
1918     my ($storecfg, $vmid, $conf, $drive) = @_;
1919
1920     if (drive_is_cloudinit($drive)) {
1921         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1922         warn $@ if $@;
1923     } elsif (!drive_is_cdrom($drive)) {
1924         my $volid = $drive->{file};
1925         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1926             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1927         }
1928     }
1929 }
1930
1931 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1932 my $smbios1_fmt = {
1933     uuid => {
1934         type => 'string',
1935         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1936         format_description => 'UUID',
1937         description => "Set SMBIOS1 UUID.",
1938         optional => 1,
1939     },
1940     version => {
1941         type => 'string',
1942         pattern => '[A-Za-z0-9+\/]+={0,2}',
1943         format_description => 'Base64 encoded string',
1944         description => "Set SMBIOS1 version.",
1945         optional => 1,
1946     },
1947     serial => {
1948         type => 'string',
1949         pattern => '[A-Za-z0-9+\/]+={0,2}',
1950         format_description => 'Base64 encoded string',
1951         description => "Set SMBIOS1 serial number.",
1952         optional => 1,
1953     },
1954     manufacturer => {
1955         type => 'string',
1956         pattern => '[A-Za-z0-9+\/]+={0,2}',
1957         format_description => 'Base64 encoded string',
1958         description => "Set SMBIOS1 manufacturer.",
1959         optional => 1,
1960     },
1961     product => {
1962         type => 'string',
1963         pattern => '[A-Za-z0-9+\/]+={0,2}',
1964         format_description => 'Base64 encoded string',
1965         description => "Set SMBIOS1 product ID.",
1966         optional => 1,
1967     },
1968     sku => {
1969         type => 'string',
1970         pattern => '[A-Za-z0-9+\/]+={0,2}',
1971         format_description => 'Base64 encoded string',
1972         description => "Set SMBIOS1 SKU string.",
1973         optional => 1,
1974     },
1975     family => {
1976         type => 'string',
1977         pattern => '[A-Za-z0-9+\/]+={0,2}',
1978         format_description => 'Base64 encoded string',
1979         description => "Set SMBIOS1 family string.",
1980         optional => 1,
1981     },
1982     base64 => {
1983         type => 'boolean',
1984         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1985         optional => 1,
1986     },
1987 };
1988
1989 sub parse_smbios1 {
1990     my ($data) = @_;
1991
1992     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1993     warn $@ if $@;
1994     return $res;
1995 }
1996
1997 sub print_smbios1 {
1998     my ($smbios1) = @_;
1999     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2000 }
2001
2002 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2003
2004 sub parse_watchdog {
2005     my ($value) = @_;
2006
2007     return if !$value;
2008
2009     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2010     warn $@ if $@;
2011     return $res;
2012 }
2013
2014 sub parse_guest_agent {
2015     my ($conf) = @_;
2016
2017     return {} if !defined($conf->{agent});
2018
2019     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2020     warn $@ if $@;
2021
2022     # if the agent is disabled ignore the other potentially set properties
2023     return {} if !$res->{enabled};
2024     return $res;
2025 }
2026
2027 sub get_qga_key {
2028     my ($conf, $key) = @_;
2029     return undef if !defined($conf->{agent});
2030
2031     my $agent = parse_guest_agent($conf);
2032     return $agent->{$key};
2033 }
2034
2035 sub parse_vga {
2036     my ($value) = @_;
2037
2038     return {} if !$value;
2039     my $res = eval { parse_property_string($vga_fmt, $value) };
2040     warn $@ if $@;
2041     return $res;
2042 }
2043
2044 sub parse_rng {
2045     my ($value) = @_;
2046
2047     return if !$value;
2048
2049     my $res = eval { parse_property_string($rng_fmt, $value) };
2050     warn $@ if $@;
2051     return $res;
2052 }
2053
2054 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2055 sub verify_usb_device {
2056     my ($value, $noerr) = @_;
2057
2058     return $value if parse_usb_device($value);
2059
2060     return if $noerr;
2061
2062     die "unable to parse usb device\n";
2063 }
2064
2065 # add JSON properties for create and set function
2066 sub json_config_properties {
2067     my $prop = shift;
2068
2069     foreach my $opt (keys %$confdesc) {
2070         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2071             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2072         $prop->{$opt} = $confdesc->{$opt};
2073     }
2074
2075     return $prop;
2076 }
2077
2078 # return copy of $confdesc_cloudinit to generate documentation
2079 sub cloudinit_config_properties {
2080
2081     return dclone($confdesc_cloudinit);
2082 }
2083
2084 sub check_type {
2085     my ($key, $value) = @_;
2086
2087     die "unknown setting '$key'\n" if !$confdesc->{$key};
2088
2089     my $type = $confdesc->{$key}->{type};
2090
2091     if (!defined($value)) {
2092         die "got undefined value\n";
2093     }
2094
2095     if ($value =~ m/[\n\r]/) {
2096         die "property contains a line feed\n";
2097     }
2098
2099     if ($type eq 'boolean') {
2100         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2101         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2102         die "type check ('boolean') failed - got '$value'\n";
2103     } elsif ($type eq 'integer') {
2104         return int($1) if $value =~ m/^(\d+)$/;
2105         die "type check ('integer') failed - got '$value'\n";
2106     } elsif ($type eq 'number') {
2107         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2108         die "type check ('number') failed - got '$value'\n";
2109     } elsif ($type eq 'string') {
2110         if (my $fmt = $confdesc->{$key}->{format}) {
2111             PVE::JSONSchema::check_format($fmt, $value);
2112             return $value;
2113         }
2114         $value =~ s/^\"(.*)\"$/$1/;
2115         return $value;
2116     } else {
2117         die "internal error"
2118     }
2119 }
2120
2121 sub destroy_vm {
2122     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2123
2124     my $conf = PVE::QemuConfig->load_config($vmid);
2125
2126     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2127
2128     if ($conf->{template}) {
2129         # check if any base image is still used by a linked clone
2130         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2131                 my ($ds, $drive) = @_;
2132                 return if drive_is_cdrom($drive);
2133
2134                 my $volid = $drive->{file};
2135                 return if !$volid || $volid =~ m|^/|;
2136
2137                 die "base volume '$volid' is still in use by linked cloned\n"
2138                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2139
2140         });
2141     }
2142
2143     my $remove_owned_drive = sub {
2144         my ($ds, $drive) = @_;
2145         return if drive_is_cdrom($drive, 1);
2146
2147         my $volid = $drive->{file};
2148         return if !$volid || $volid =~ m|^/|;
2149
2150         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2151         return if !$path || !$owner || ($owner != $vmid);
2152
2153         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2154         warn "Could not remove disk '$volid', check manually: $@" if $@;
2155     };
2156
2157     # only remove disks owned by this VM (referenced in the config)
2158     my $include_opts = {
2159         include_unused => 1,
2160         extra_keys => ['vmstate'],
2161     };
2162     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2163
2164     for my $snap (values %{$conf->{snapshots}}) {
2165         next if !defined($snap->{vmstate});
2166         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2167         next if !defined($drive);
2168         $remove_owned_drive->('vmstate', $drive);
2169     }
2170
2171     if ($purge_unreferenced) { # also remove unreferenced disk
2172         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2173         PVE::Storage::foreach_volid($vmdisks, sub {
2174             my ($volid, $sid, $volname, $d) = @_;
2175             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2176             warn $@ if $@;
2177         });
2178     }
2179
2180     if (defined $replacement_conf) {
2181         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2182     } else {
2183         PVE::QemuConfig->destroy_config($vmid);
2184     }
2185 }
2186
2187 sub parse_vm_config {
2188     my ($filename, $raw) = @_;
2189
2190     return if !defined($raw);
2191
2192     my $res = {
2193         digest => Digest::SHA::sha1_hex($raw),
2194         snapshots => {},
2195         pending => {},
2196     };
2197
2198     $filename =~ m|/qemu-server/(\d+)\.conf$|
2199         || die "got strange filename '$filename'";
2200
2201     my $vmid = $1;
2202
2203     my $conf = $res;
2204     my $descr;
2205     my $section = '';
2206
2207     my @lines = split(/\n/, $raw);
2208     foreach my $line (@lines) {
2209         next if $line =~ m/^\s*$/;
2210
2211         if ($line =~ m/^\[PENDING\]\s*$/i) {
2212             $section = 'pending';
2213             if (defined($descr)) {
2214                 $descr =~ s/\s+$//;
2215                 $conf->{description} = $descr;
2216             }
2217             $descr = undef;
2218             $conf = $res->{$section} = {};
2219             next;
2220
2221         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2222             $section = $1;
2223             if (defined($descr)) {
2224                 $descr =~ s/\s+$//;
2225                 $conf->{description} = $descr;
2226             }
2227             $descr = undef;
2228             $conf = $res->{snapshots}->{$section} = {};
2229             next;
2230         }
2231
2232         if ($line =~ m/^\#(.*)\s*$/) {
2233             $descr = '' if !defined($descr);
2234             $descr .= PVE::Tools::decode_text($1) . "\n";
2235             next;
2236         }
2237
2238         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2239             $descr = '' if !defined($descr);
2240             $descr .= PVE::Tools::decode_text($2);
2241         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2242             $conf->{snapstate} = $1;
2243         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2244             my $key = $1;
2245             my $value = $2;
2246             $conf->{$key} = $value;
2247         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2248             my $value = $1;
2249             if ($section eq 'pending') {
2250                 $conf->{delete} = $value; # we parse this later
2251             } else {
2252                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2253             }
2254         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2255             my $key = $1;
2256             my $value = $2;
2257             eval { $value = check_type($key, $value); };
2258             if ($@) {
2259                 warn "vm $vmid - unable to parse value of '$key' - $@";
2260             } else {
2261                 $key = 'ide2' if $key eq 'cdrom';
2262                 my $fmt = $confdesc->{$key}->{format};
2263                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2264                     my $v = parse_drive($key, $value);
2265                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2266                         $v->{file} = $volid;
2267                         $value = print_drive($v);
2268                     } else {
2269                         warn "vm $vmid - unable to parse value of '$key'\n";
2270                         next;
2271                     }
2272                 }
2273
2274                 $conf->{$key} = $value;
2275             }
2276         } else {
2277             warn "vm $vmid - unable to parse config: $line\n";
2278         }
2279     }
2280
2281     if (defined($descr)) {
2282         $descr =~ s/\s+$//;
2283         $conf->{description} = $descr;
2284     }
2285     delete $res->{snapstate}; # just to be sure
2286
2287     return $res;
2288 }
2289
2290 sub write_vm_config {
2291     my ($filename, $conf) = @_;
2292
2293     delete $conf->{snapstate}; # just to be sure
2294
2295     if ($conf->{cdrom}) {
2296         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2297         $conf->{ide2} = $conf->{cdrom};
2298         delete $conf->{cdrom};
2299     }
2300
2301     # we do not use 'smp' any longer
2302     if ($conf->{sockets}) {
2303         delete $conf->{smp};
2304     } elsif ($conf->{smp}) {
2305         $conf->{sockets} = $conf->{smp};
2306         delete $conf->{cores};
2307         delete $conf->{smp};
2308     }
2309
2310     my $used_volids = {};
2311
2312     my $cleanup_config = sub {
2313         my ($cref, $pending, $snapname) = @_;
2314
2315         foreach my $key (keys %$cref) {
2316             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2317                 $key eq 'snapstate' || $key eq 'pending';
2318             my $value = $cref->{$key};
2319             if ($key eq 'delete') {
2320                 die "propertry 'delete' is only allowed in [PENDING]\n"
2321                     if !$pending;
2322                 # fixme: check syntax?
2323                 next;
2324             }
2325             eval { $value = check_type($key, $value); };
2326             die "unable to parse value of '$key' - $@" if $@;
2327
2328             $cref->{$key} = $value;
2329
2330             if (!$snapname && is_valid_drivename($key)) {
2331                 my $drive = parse_drive($key, $value);
2332                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2333             }
2334         }
2335     };
2336
2337     &$cleanup_config($conf);
2338
2339     &$cleanup_config($conf->{pending}, 1);
2340
2341     foreach my $snapname (keys %{$conf->{snapshots}}) {
2342         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2343         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2344     }
2345
2346     # remove 'unusedX' settings if we re-add a volume
2347     foreach my $key (keys %$conf) {
2348         my $value = $conf->{$key};
2349         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2350             delete $conf->{$key};
2351         }
2352     }
2353
2354     my $generate_raw_config = sub {
2355         my ($conf, $pending) = @_;
2356
2357         my $raw = '';
2358
2359         # add description as comment to top of file
2360         if (defined(my $descr = $conf->{description})) {
2361             if ($descr) {
2362                 foreach my $cl (split(/\n/, $descr)) {
2363                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2364                 }
2365             } else {
2366                 $raw .= "#\n" if $pending;
2367             }
2368         }
2369
2370         foreach my $key (sort keys %$conf) {
2371             next if $key =~ /^(digest|description|pending|snapshots)$/;
2372             $raw .= "$key: $conf->{$key}\n";
2373         }
2374         return $raw;
2375     };
2376
2377     my $raw = &$generate_raw_config($conf);
2378
2379     if (scalar(keys %{$conf->{pending}})){
2380         $raw .= "\n[PENDING]\n";
2381         $raw .= &$generate_raw_config($conf->{pending}, 1);
2382     }
2383
2384     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2385         $raw .= "\n[$snapname]\n";
2386         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2387     }
2388
2389     return $raw;
2390 }
2391
2392 sub load_defaults {
2393
2394     my $res = {};
2395
2396     # we use static defaults from our JSON schema configuration
2397     foreach my $key (keys %$confdesc) {
2398         if (defined(my $default = $confdesc->{$key}->{default})) {
2399             $res->{$key} = $default;
2400         }
2401     }
2402
2403     return $res;
2404 }
2405
2406 sub config_list {
2407     my $vmlist = PVE::Cluster::get_vmlist();
2408     my $res = {};
2409     return $res if !$vmlist || !$vmlist->{ids};
2410     my $ids = $vmlist->{ids};
2411     my $nodename = nodename();
2412
2413     foreach my $vmid (keys %$ids) {
2414         my $d = $ids->{$vmid};
2415         next if !$d->{node} || $d->{node} ne $nodename;
2416         next if !$d->{type} || $d->{type} ne 'qemu';
2417         $res->{$vmid}->{exists} = 1;
2418     }
2419     return $res;
2420 }
2421
2422 # test if VM uses local resources (to prevent migration)
2423 sub check_local_resources {
2424     my ($conf, $noerr) = @_;
2425
2426     my @loc_res = ();
2427
2428     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2429     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2430
2431     push @loc_res, "ivshmem" if $conf->{ivshmem};
2432
2433     foreach my $k (keys %$conf) {
2434         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2435         # sockets are safe: they will recreated be on the target side post-migrate
2436         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2437         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2438     }
2439
2440     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2441
2442     return \@loc_res;
2443 }
2444
2445 # check if used storages are available on all nodes (use by migrate)
2446 sub check_storage_availability {
2447     my ($storecfg, $conf, $node) = @_;
2448
2449     PVE::QemuConfig->foreach_volume($conf, sub {
2450         my ($ds, $drive) = @_;
2451
2452         my $volid = $drive->{file};
2453         return if !$volid;
2454
2455         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2456         return if !$sid;
2457
2458         # check if storage is available on both nodes
2459         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2460         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2461    });
2462 }
2463
2464 # list nodes where all VM images are available (used by has_feature API)
2465 sub shared_nodes {
2466     my ($conf, $storecfg) = @_;
2467
2468     my $nodelist = PVE::Cluster::get_nodelist();
2469     my $nodehash = { map { $_ => 1 } @$nodelist };
2470     my $nodename = nodename();
2471
2472     PVE::QemuConfig->foreach_volume($conf, sub {
2473         my ($ds, $drive) = @_;
2474
2475         my $volid = $drive->{file};
2476         return if !$volid;
2477
2478         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2479         if ($storeid) {
2480             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2481             if ($scfg->{disable}) {
2482                 $nodehash = {};
2483             } elsif (my $avail = $scfg->{nodes}) {
2484                 foreach my $node (keys %$nodehash) {
2485                     delete $nodehash->{$node} if !$avail->{$node};
2486                 }
2487             } elsif (!$scfg->{shared}) {
2488                 foreach my $node (keys %$nodehash) {
2489                     delete $nodehash->{$node} if $node ne $nodename
2490                 }
2491             }
2492         }
2493     });
2494
2495     return $nodehash
2496 }
2497
2498 sub check_local_storage_availability {
2499     my ($conf, $storecfg) = @_;
2500
2501     my $nodelist = PVE::Cluster::get_nodelist();
2502     my $nodehash = { map { $_ => {} } @$nodelist };
2503
2504     PVE::QemuConfig->foreach_volume($conf, sub {
2505         my ($ds, $drive) = @_;
2506
2507         my $volid = $drive->{file};
2508         return if !$volid;
2509
2510         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2511         if ($storeid) {
2512             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2513
2514             if ($scfg->{disable}) {
2515                 foreach my $node (keys %$nodehash) {
2516                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2517                 }
2518             } elsif (my $avail = $scfg->{nodes}) {
2519                 foreach my $node (keys %$nodehash) {
2520                     if (!$avail->{$node}) {
2521                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2522                     }
2523                 }
2524             }
2525         }
2526     });
2527
2528     foreach my $node (values %$nodehash) {
2529         if (my $unavail = $node->{unavailable_storages}) {
2530             $node->{unavailable_storages} = [ sort keys %$unavail ];
2531         }
2532     }
2533
2534     return $nodehash
2535 }
2536
2537 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2538 sub check_running {
2539     my ($vmid, $nocheck, $node) = @_;
2540
2541     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2542     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2543 }
2544
2545 sub vzlist {
2546
2547     my $vzlist = config_list();
2548
2549     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2550
2551     while (defined(my $de = $fd->read)) {
2552         next if $de !~ m/^(\d+)\.pid$/;
2553         my $vmid = $1;
2554         next if !defined($vzlist->{$vmid});
2555         if (my $pid = check_running($vmid)) {
2556             $vzlist->{$vmid}->{pid} = $pid;
2557         }
2558     }
2559
2560     return $vzlist;
2561 }
2562
2563 our $vmstatus_return_properties = {
2564     vmid => get_standard_option('pve-vmid'),
2565     status => {
2566         description => "Qemu process status.",
2567         type => 'string',
2568         enum => ['stopped', 'running'],
2569     },
2570     maxmem => {
2571         description => "Maximum memory in bytes.",
2572         type => 'integer',
2573         optional => 1,
2574         renderer => 'bytes',
2575     },
2576     maxdisk => {
2577         description => "Root disk size in bytes.",
2578         type => 'integer',
2579         optional => 1,
2580         renderer => 'bytes',
2581     },
2582     name => {
2583         description => "VM name.",
2584         type => 'string',
2585         optional => 1,
2586     },
2587     qmpstatus => {
2588         description => "Qemu QMP agent status.",
2589         type => 'string',
2590         optional => 1,
2591     },
2592     pid => {
2593         description => "PID of running qemu process.",
2594         type => 'integer',
2595         optional => 1,
2596     },
2597     uptime => {
2598         description => "Uptime.",
2599         type => 'integer',
2600         optional => 1,
2601         renderer => 'duration',
2602     },
2603     cpus => {
2604         description => "Maximum usable CPUs.",
2605         type => 'number',
2606         optional => 1,
2607     },
2608     lock => {
2609         description => "The current config lock, if any.",
2610         type => 'string',
2611         optional => 1,
2612     },
2613     tags => {
2614         description  => "The current configured tags, if any",
2615         type => 'string',
2616         optional => 1,
2617     },
2618     'running-machine' => {
2619         description => "The currently running machine type (if running).",
2620         type => 'string',
2621         optional => 1,
2622     },
2623     'running-qemu' => {
2624         description => "The currently running QEMU version (if running).",
2625         type => 'string',
2626         optional => 1,
2627     },
2628 };
2629
2630 my $last_proc_pid_stat;
2631
2632 # get VM status information
2633 # This must be fast and should not block ($full == false)
2634 # We only query KVM using QMP if $full == true (this can be slow)
2635 sub vmstatus {
2636     my ($opt_vmid, $full) = @_;
2637
2638     my $res = {};
2639
2640     my $storecfg = PVE::Storage::config();
2641
2642     my $list = vzlist();
2643     my $defaults = load_defaults();
2644
2645     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2646
2647     my $cpucount = $cpuinfo->{cpus} || 1;
2648
2649     foreach my $vmid (keys %$list) {
2650         next if $opt_vmid && ($vmid ne $opt_vmid);
2651
2652         my $conf = PVE::QemuConfig->load_config($vmid);
2653
2654         my $d = { vmid => int($vmid) };
2655         $d->{pid} = int($list->{$vmid}->{pid});
2656
2657         # fixme: better status?
2658         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2659
2660         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2661         if (defined($size)) {
2662             $d->{disk} = 0; # no info available
2663             $d->{maxdisk} = $size;
2664         } else {
2665             $d->{disk} = 0;
2666             $d->{maxdisk} = 0;
2667         }
2668
2669         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2670             * ($conf->{cores} || $defaults->{cores});
2671         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2672         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2673
2674         $d->{name} = $conf->{name} || "VM $vmid";
2675         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2676             : $defaults->{memory}*(1024*1024);
2677
2678         if ($conf->{balloon}) {
2679             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2680             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2681                 : $defaults->{shares};
2682         }
2683
2684         $d->{uptime} = 0;
2685         $d->{cpu} = 0;
2686         $d->{mem} = 0;
2687
2688         $d->{netout} = 0;
2689         $d->{netin} = 0;
2690
2691         $d->{diskread} = 0;
2692         $d->{diskwrite} = 0;
2693
2694         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2695
2696         $d->{serial} = 1 if conf_has_serial($conf);
2697         $d->{lock} = $conf->{lock} if $conf->{lock};
2698         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2699
2700         $res->{$vmid} = $d;
2701     }
2702
2703     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2704     foreach my $dev (keys %$netdev) {
2705         next if $dev !~ m/^tap([1-9]\d*)i/;
2706         my $vmid = $1;
2707         my $d = $res->{$vmid};
2708         next if !$d;
2709
2710         $d->{netout} += $netdev->{$dev}->{receive};
2711         $d->{netin} += $netdev->{$dev}->{transmit};
2712
2713         if ($full) {
2714             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2715             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2716         }
2717
2718     }
2719
2720     my $ctime = gettimeofday;
2721
2722     foreach my $vmid (keys %$list) {
2723
2724         my $d = $res->{$vmid};
2725         my $pid = $d->{pid};
2726         next if !$pid;
2727
2728         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2729         next if !$pstat; # not running
2730
2731         my $used = $pstat->{utime} + $pstat->{stime};
2732
2733         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2734
2735         if ($pstat->{vsize}) {
2736             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2737         }
2738
2739         my $old = $last_proc_pid_stat->{$pid};
2740         if (!$old) {
2741             $last_proc_pid_stat->{$pid} = {
2742                 time => $ctime,
2743                 used => $used,
2744                 cpu => 0,
2745             };
2746             next;
2747         }
2748
2749         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2750
2751         if ($dtime > 1000) {
2752             my $dutime = $used -  $old->{used};
2753
2754             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2755             $last_proc_pid_stat->{$pid} = {
2756                 time => $ctime,
2757                 used => $used,
2758                 cpu => $d->{cpu},
2759             };
2760         } else {
2761             $d->{cpu} = $old->{cpu};
2762         }
2763     }
2764
2765     return $res if !$full;
2766
2767     my $qmpclient = PVE::QMPClient->new();
2768
2769     my $ballooncb = sub {
2770         my ($vmid, $resp) = @_;
2771
2772         my $info = $resp->{'return'};
2773         return if !$info->{max_mem};
2774
2775         my $d = $res->{$vmid};
2776
2777         # use memory assigned to VM
2778         $d->{maxmem} = $info->{max_mem};
2779         $d->{balloon} = $info->{actual};
2780
2781         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2782             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2783             $d->{freemem} = $info->{free_mem};
2784         }
2785
2786         $d->{ballooninfo} = $info;
2787     };
2788
2789     my $blockstatscb = sub {
2790         my ($vmid, $resp) = @_;
2791         my $data = $resp->{'return'} || [];
2792         my $totalrdbytes = 0;
2793         my $totalwrbytes = 0;
2794
2795         for my $blockstat (@$data) {
2796             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2797             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2798
2799             $blockstat->{device} =~ s/drive-//;
2800             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2801         }
2802         $res->{$vmid}->{diskread} = $totalrdbytes;
2803         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2804     };
2805
2806     my $machinecb = sub {
2807         my ($vmid, $resp) = @_;
2808         my $data = $resp->{'return'} || [];
2809
2810         $res->{$vmid}->{'running-machine'} =
2811             PVE::QemuServer::Machine::current_from_query_machines($data);
2812     };
2813
2814     my $versioncb = sub {
2815         my ($vmid, $resp) = @_;
2816         my $data = $resp->{'return'} // {};
2817         my $version = 'unknown';
2818
2819         if (my $v = $data->{qemu}) {
2820             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2821         }
2822
2823         $res->{$vmid}->{'running-qemu'} = $version;
2824     };
2825
2826     my $statuscb = sub {
2827         my ($vmid, $resp) = @_;
2828
2829         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2830         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2831         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2832         # this fails if ballon driver is not loaded, so this must be
2833         # the last commnand (following command are aborted if this fails).
2834         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2835
2836         my $status = 'unknown';
2837         if (!defined($status = $resp->{'return'}->{status})) {
2838             warn "unable to get VM status\n";
2839             return;
2840         }
2841
2842         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2843     };
2844
2845     foreach my $vmid (keys %$list) {
2846         next if $opt_vmid && ($vmid ne $opt_vmid);
2847         next if !$res->{$vmid}->{pid}; # not running
2848         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2849     }
2850
2851     $qmpclient->queue_execute(undef, 2);
2852
2853     foreach my $vmid (keys %$list) {
2854         next if $opt_vmid && ($vmid ne $opt_vmid);
2855         next if !$res->{$vmid}->{pid}; #not running
2856
2857         # we can't use the $qmpclient since it might have already aborted on
2858         # 'query-balloon', but this might also fail for older versions...
2859         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2860         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2861     }
2862
2863     foreach my $vmid (keys %$list) {
2864         next if $opt_vmid && ($vmid ne $opt_vmid);
2865         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2866     }
2867
2868     return $res;
2869 }
2870
2871 sub conf_has_serial {
2872     my ($conf) = @_;
2873
2874     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2875         if ($conf->{"serial$i"}) {
2876             return 1;
2877         }
2878     }
2879
2880     return 0;
2881 }
2882
2883 sub conf_has_audio {
2884     my ($conf, $id) = @_;
2885
2886     $id //= 0;
2887     my $audio = $conf->{"audio$id"};
2888     return if !defined($audio);
2889
2890     my $audioproperties = parse_property_string($audio_fmt, $audio);
2891     my $audiodriver = $audioproperties->{driver} // 'spice';
2892
2893     return {
2894         dev => $audioproperties->{device},
2895         dev_id => "audiodev$id",
2896         backend => $audiodriver,
2897         backend_id => "$audiodriver-backend${id}",
2898     };
2899 }
2900
2901 sub audio_devs {
2902     my ($audio, $audiopciaddr, $machine_version) = @_;
2903
2904     my $devs = [];
2905
2906     my $id = $audio->{dev_id};
2907     my $audiodev = "";
2908     if (min_version($machine_version, 4, 2)) {
2909         $audiodev = ",audiodev=$audio->{backend_id}";
2910     }
2911
2912     if ($audio->{dev} eq 'AC97') {
2913         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2914     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2915         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2916         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2917         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2918     } else {
2919         die "unkown audio device '$audio->{dev}', implement me!";
2920     }
2921
2922     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2923
2924     return $devs;
2925 }
2926
2927 sub vga_conf_has_spice {
2928     my ($vga) = @_;
2929
2930     my $vgaconf = parse_vga($vga);
2931     my $vgatype = $vgaconf->{type};
2932     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2933
2934     return $1 || 1;
2935 }
2936
2937 sub is_native($) {
2938     my ($arch) = @_;
2939     return get_host_arch() eq $arch;
2940 }
2941
2942 sub get_vm_arch {
2943     my ($conf) = @_;
2944     return $conf->{arch} // get_host_arch();
2945 }
2946
2947 my $default_machines = {
2948     x86_64 => 'pc',
2949     aarch64 => 'virt',
2950 };
2951
2952 sub get_installed_machine_version {
2953     my ($kvmversion) = @_;
2954     $kvmversion = kvm_user_version() if !defined($kvmversion);
2955     $kvmversion =~ m/^(\d+\.\d+)/;
2956     return $1;
2957 }
2958
2959 sub windows_get_pinned_machine_version {
2960     my ($machine, $base_version, $kvmversion) = @_;
2961
2962     my $pin_version = $base_version;
2963     if (!defined($base_version) ||
2964         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2965     ) {
2966         $pin_version = get_installed_machine_version($kvmversion);
2967     }
2968     if (!$machine || $machine eq 'pc') {
2969         $machine = "pc-i440fx-$pin_version";
2970     } elsif ($machine eq 'q35') {
2971         $machine = "pc-q35-$pin_version";
2972     } elsif ($machine eq 'virt') {
2973         $machine = "virt-$pin_version";
2974     } else {
2975         warn "unknown machine type '$machine', not touching that!\n";
2976     }
2977
2978     return $machine;
2979 }
2980
2981 sub get_vm_machine {
2982     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2983
2984     my $machine = $forcemachine || $conf->{machine};
2985
2986     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2987         $kvmversion //= kvm_user_version();
2988         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2989         # layout which confuses windows quite a bit and may result in various regressions..
2990         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2991         if (windows_version($conf->{ostype})) {
2992             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2993         }
2994         $arch //= 'x86_64';
2995         $machine ||= $default_machines->{$arch};
2996         if ($add_pve_version) {
2997             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2998             $machine .= "+pve$pvever";
2999         }
3000     }
3001
3002     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3003         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3004         $machine = $1 if $is_pxe;
3005
3006         # for version-pinned machines that do not include a pve-version (e.g.
3007         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3008         $machine .= '+pve0';
3009
3010         $machine .= '.pxe' if $is_pxe;
3011     }
3012
3013     return $machine;
3014 }
3015
3016 sub get_ovmf_files($) {
3017     my ($arch) = @_;
3018
3019     my $ovmf = $OVMF->{$arch}
3020         or die "no OVMF images known for architecture '$arch'\n";
3021
3022     return @$ovmf;
3023 }
3024
3025 my $Arch2Qemu = {
3026     aarch64 => '/usr/bin/qemu-system-aarch64',
3027     x86_64 => '/usr/bin/qemu-system-x86_64',
3028 };
3029 sub get_command_for_arch($) {
3030     my ($arch) = @_;
3031     return '/usr/bin/kvm' if is_native($arch);
3032
3033     my $cmd = $Arch2Qemu->{$arch}
3034         or die "don't know how to emulate architecture '$arch'\n";
3035     return $cmd;
3036 }
3037
3038 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3039 # to use in a QEMU command line (-cpu element), first array_intersect the result
3040 # of query_supported_ with query_understood_. This is necessary because:
3041 #
3042 # a) query_understood_ returns flags the host cannot use and
3043 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3044 #    flags, but CPU settings - with most of them being flags. Those settings
3045 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3046 #
3047 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3048 # expensive. If you need the value returned from this, you can get it much
3049 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3050 # $accel being 'kvm' or 'tcg'.
3051 #
3052 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3053 # changes, automatically populating pmxcfs.
3054 #
3055 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3056 # since kvm and tcg machines support different flags
3057 #
3058 sub query_supported_cpu_flags {
3059     my ($arch) = @_;
3060
3061     $arch //= get_host_arch();
3062     my $default_machine = $default_machines->{$arch};
3063
3064     my $flags = {};
3065
3066     # FIXME: Once this is merged, the code below should work for ARM as well:
3067     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3068     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3069         $arch eq "aarch64";
3070
3071     my $kvm_supported = defined(kvm_version());
3072     my $qemu_cmd = get_command_for_arch($arch);
3073     my $fakevmid = -1;
3074     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3075
3076     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3077     my $query_supported_run_qemu = sub {
3078         my ($kvm) = @_;
3079
3080         my $flags = {};
3081         my $cmd = [
3082             $qemu_cmd,
3083             '-machine', $default_machine,
3084             '-display', 'none',
3085             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3086             '-mon', 'chardev=qmp,mode=control',
3087             '-pidfile', $pidfile,
3088             '-S', '-daemonize'
3089         ];
3090
3091         if (!$kvm) {
3092             push @$cmd, '-accel', 'tcg';
3093         }
3094
3095         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3096         die "QEMU flag querying VM exited with code " . $rc if $rc;
3097
3098         eval {
3099             my $cmd_result = mon_cmd(
3100                 $fakevmid,
3101                 'query-cpu-model-expansion',
3102                 type => 'full',
3103                 model => { name => 'host' }
3104             );
3105
3106             my $props = $cmd_result->{model}->{props};
3107             foreach my $prop (keys %$props) {
3108                 next if $props->{$prop} ne '1';
3109                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3110                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3111                 # We only keep those with underscores, to match /proc/cpuinfo
3112                 $prop =~ s/\.|-/_/g;
3113                 $flags->{$prop} = 1;
3114             }
3115         };
3116         my $err = $@;
3117
3118         # force stop with 10 sec timeout and 'nocheck'
3119         # always stop, even if QMP failed
3120         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3121
3122         die $err if $err;
3123
3124         return [ sort keys %$flags ];
3125     };
3126
3127     # We need to query QEMU twice, since KVM and TCG have different supported flags
3128     PVE::QemuConfig->lock_config($fakevmid, sub {
3129         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3130         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3131
3132         if ($kvm_supported) {
3133             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3134             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3135         }
3136     });
3137
3138     return $flags;
3139 }
3140
3141 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3142 my $understood_cpu_flag_dir = "/usr/share/kvm";
3143 sub query_understood_cpu_flags {
3144     my $arch = get_host_arch();
3145     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3146
3147     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3148         if ! -e $filepath;
3149
3150     my $raw = file_get_contents($filepath);
3151     $raw =~ s/^\s+|\s+$//g;
3152     my @flags = split(/\s+/, $raw);
3153
3154     return \@flags;
3155 }
3156
3157 sub config_to_command {
3158     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3159         $pbs_backing) = @_;
3160
3161     my $cmd = [];
3162     my $globalFlags = [];
3163     my $machineFlags = [];
3164     my $rtcFlags = [];
3165     my $devices = [];
3166     my $pciaddr = '';
3167     my $bridges = {};
3168     my $ostype = $conf->{ostype};
3169     my $winversion = windows_version($ostype);
3170     my $kvm = $conf->{kvm};
3171     my $nodename = nodename();
3172
3173     my $arch = get_vm_arch($conf);
3174     my $kvm_binary = get_command_for_arch($arch);
3175     my $kvmver = kvm_user_version($kvm_binary);
3176
3177     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3178         $kvmver //= "undefined";
3179         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3180     }
3181
3182     my $add_pve_version = min_version($kvmver, 4, 1);
3183
3184     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3185     my $machine_version = extract_version($machine_type, $kvmver);
3186     $kvm //= 1 if is_native($arch);
3187
3188     $machine_version =~ m/(\d+)\.(\d+)/;
3189     my ($machine_major, $machine_minor) = ($1, $2);
3190
3191     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3192         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3193     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3194         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3195             ." please upgrade node '$nodename'\n"
3196     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3197         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3198         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3199             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3200             ." node '$nodename'\n";
3201     }
3202
3203     # if a specific +pve version is required for a feature, use $version_guard
3204     # instead of min_version to allow machines to be run with the minimum
3205     # required version
3206     my $required_pve_version = 0;
3207     my $version_guard = sub {
3208         my ($major, $minor, $pve) = @_;
3209         return 0 if !min_version($machine_version, $major, $minor, $pve);
3210         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3211         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3212         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3213         return 1;
3214     };
3215
3216     if ($kvm && !defined kvm_version()) {
3217         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3218             ." or enable in BIOS.\n";
3219     }
3220
3221     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3222     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3223     my $use_old_bios_files = undef;
3224     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3225
3226     my $cpuunits = defined($conf->{cpuunits}) ?
3227             $conf->{cpuunits} : $defaults->{cpuunits};
3228
3229     push @$cmd, $kvm_binary;
3230
3231     push @$cmd, '-id', $vmid;
3232
3233     my $vmname = $conf->{name} || "vm$vmid";
3234
3235     push @$cmd, '-name', $vmname;
3236
3237     push @$cmd, '-no-shutdown';
3238
3239     my $use_virtio = 0;
3240
3241     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3242     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3243     push @$cmd, '-mon', "chardev=qmp,mode=control";
3244
3245     if (min_version($machine_version, 2, 12)) {
3246         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3247         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3248     }
3249
3250     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3251
3252     push @$cmd, '-daemonize';
3253
3254     if ($conf->{smbios1}) {
3255         my $smbios_conf = parse_smbios1($conf->{smbios1});
3256         if ($smbios_conf->{base64}) {
3257             # Do not pass base64 flag to qemu
3258             delete $smbios_conf->{base64};
3259             my $smbios_string = "";
3260             foreach my $key (keys %$smbios_conf) {
3261                 my $value;
3262                 if ($key eq "uuid") {
3263                     $value = $smbios_conf->{uuid}
3264                 } else {
3265                     $value = decode_base64($smbios_conf->{$key});
3266                 }
3267                 # qemu accepts any binary data, only commas need escaping by double comma
3268                 $value =~ s/,/,,/g;
3269                 $smbios_string .= "," . $key . "=" . $value if $value;
3270             }
3271             push @$cmd, '-smbios', "type=1" . $smbios_string;
3272         } else {
3273             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3274         }
3275     }
3276
3277     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3278         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3279         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3280
3281         my ($path, $format);
3282         if (my $efidisk = $conf->{efidisk0}) {
3283             my $d = parse_drive('efidisk0', $efidisk);
3284             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3285             $format = $d->{format};
3286             if ($storeid) {
3287                 $path = PVE::Storage::path($storecfg, $d->{file});
3288                 if (!defined($format)) {
3289                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3290                     $format = qemu_img_format($scfg, $volname);
3291                 }
3292             } else {
3293                 $path = $d->{file};
3294                 die "efidisk format must be specified\n"
3295                     if !defined($format);
3296             }
3297         } else {
3298             warn "no efidisk configured! Using temporary efivars disk.\n";
3299             $path = "/tmp/$vmid-ovmf.fd";
3300             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3301             $format = 'raw';
3302         }
3303
3304         my $size_str = "";
3305
3306         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3307             $size_str = ",size=" . (-s $ovmf_vars);
3308         }
3309
3310         # on slower ceph clusters, booting without cache on efidisk can take a while, see #3329
3311         my $cache = $path =~ m/^rbd:/ ? ',cache=writeback' : '';
3312
3313         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3314         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=$path";
3315     }
3316
3317     # load q35 config
3318     if ($q35) {
3319         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3320         if (min_version($machine_version, 4, 0)) {
3321             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3322         } else {
3323             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3324         }
3325     }
3326
3327     if ($conf->{vmgenid}) {
3328         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3329     }
3330
3331     # add usb controllers
3332     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3333         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3334     push @$devices, @usbcontrollers if @usbcontrollers;
3335     my $vga = parse_vga($conf->{vga});
3336
3337     my $qxlnum = vga_conf_has_spice($conf->{vga});
3338     $vga->{type} = 'qxl' if $qxlnum;
3339
3340     if (!$vga->{type}) {
3341         if ($arch eq 'aarch64') {
3342             $vga->{type} = 'virtio';
3343         } elsif (min_version($machine_version, 2, 9)) {
3344             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3345         } else {
3346             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3347         }
3348     }
3349
3350     # enable absolute mouse coordinates (needed by vnc)
3351     my $tablet;
3352     if (defined($conf->{tablet})) {
3353         $tablet = $conf->{tablet};
3354     } else {
3355         $tablet = $defaults->{tablet};
3356         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3357         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3358     }
3359
3360     if ($tablet) {
3361         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3362         my $kbd = print_keyboarddevice_full($conf, $arch);
3363         push @$devices, '-device', $kbd if defined($kbd);
3364     }
3365
3366     my $bootorder = device_bootorder($conf);
3367
3368     # host pci device passthrough
3369     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3370         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3371
3372     # usb devices
3373     my $usb_dev_features = {};
3374     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3375
3376     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3377         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3378     push @$devices, @usbdevices if @usbdevices;
3379
3380     # serial devices
3381     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3382         if (my $path = $conf->{"serial$i"}) {
3383             if ($path eq 'socket') {
3384                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3385                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3386                 # On aarch64, serial0 is the UART device. Qemu only allows
3387                 # connecting UART devices via the '-serial' command line, as
3388                 # the device has a fixed slot on the hardware...
3389                 if ($arch eq 'aarch64' && $i == 0) {
3390                     push @$devices, '-serial', "chardev:serial$i";
3391                 } else {
3392                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3393                 }
3394             } else {
3395                 die "no such serial device\n" if ! -c $path;
3396                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3397                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3398             }
3399         }
3400     }
3401
3402     # parallel devices
3403     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3404         if (my $path = $conf->{"parallel$i"}) {
3405             die "no such parallel device\n" if ! -c $path;
3406             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3407             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3408             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3409         }
3410     }
3411
3412     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3413         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3414         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3415         push @$devices, @$audio_devs;
3416     }
3417
3418     my $sockets = 1;
3419     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3420     $sockets = $conf->{sockets} if  $conf->{sockets};
3421
3422     my $cores = $conf->{cores} || 1;
3423
3424     my $maxcpus = $sockets * $cores;
3425
3426     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3427
3428     my $allowed_vcpus = $cpuinfo->{cpus};
3429
3430     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3431         if ($allowed_vcpus < $maxcpus);
3432
3433     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3434
3435         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3436         for (my $i = 2; $i <= $vcpus; $i++)  {
3437             my $cpustr = print_cpu_device($conf,$i);
3438             push @$cmd, '-device', $cpustr;
3439         }
3440
3441     } else {
3442
3443         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3444     }
3445     push @$cmd, '-nodefaults';
3446
3447     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3448
3449     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3450
3451     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3452
3453     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3454         push @$devices, '-device', print_vga_device(
3455             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3456         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3457         push @$cmd,  '-vnc', "unix:$socket,password=on";
3458     } else {
3459         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3460         push @$cmd, '-nographic';
3461     }
3462
3463     # time drift fix
3464     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3465     my $useLocaltime = $conf->{localtime};
3466
3467     if ($winversion >= 5) { # windows
3468         $useLocaltime = 1 if !defined($conf->{localtime});
3469
3470         # use time drift fix when acpi is enabled
3471         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3472             $tdf = 1 if !defined($conf->{tdf});
3473         }
3474     }
3475
3476     if ($winversion >= 6) {
3477         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3478         push @$cmd, '-no-hpet';
3479     }
3480
3481     push @$rtcFlags, 'driftfix=slew' if $tdf;
3482
3483     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3484         push @$rtcFlags, "base=$conf->{startdate}";
3485     } elsif ($useLocaltime) {
3486         push @$rtcFlags, 'base=localtime';
3487     }
3488
3489     if ($forcecpu) {
3490         push @$cmd, '-cpu', $forcecpu;
3491     } else {
3492         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3493     }
3494
3495     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3496
3497     push @$cmd, '-S' if $conf->{freeze};
3498
3499     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3500
3501     my $guest_agent = parse_guest_agent($conf);
3502
3503     if ($guest_agent->{enabled}) {
3504         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3505         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3506
3507         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3508             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3509             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3510             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3511         } elsif ($guest_agent->{type} eq 'isa') {
3512             push @$devices, '-device', "isa-serial,chardev=qga0";
3513         }
3514     }
3515
3516     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3517     if ($rng && $version_guard->(4, 1, 2)) {
3518         check_rng_source($rng->{source});
3519
3520         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3521         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3522         my $limiter_str = "";
3523         if ($max_bytes) {
3524             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3525         }
3526
3527         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3528         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3529         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3530     }
3531
3532     my $spice_port;
3533
3534     if ($qxlnum) {
3535         if ($qxlnum > 1) {
3536             if ($winversion){
3537                 for (my $i = 1; $i < $qxlnum; $i++){
3538                     push @$devices, '-device', print_vga_device(
3539                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3540                 }
3541             } else {
3542                 # assume other OS works like Linux
3543                 my ($ram, $vram) = ("134217728", "67108864");
3544                 if ($vga->{memory}) {
3545                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3546                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3547                 }
3548                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3549                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3550             }
3551         }
3552
3553         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3554
3555         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3556         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3557         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3558
3559         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3560         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3561         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3562
3563         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3564         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3565
3566         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3567         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3568         if ($spice_enhancement->{foldersharing}) {
3569             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3570             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3571         }
3572
3573         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3574         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3575             if $spice_enhancement->{videostreaming};
3576
3577         push @$devices, '-spice', "$spice_opts";
3578     }
3579
3580     # enable balloon by default, unless explicitly disabled
3581     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3582         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3583         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3584     }
3585
3586     if ($conf->{watchdog}) {
3587         my $wdopts = parse_watchdog($conf->{watchdog});
3588         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3589         my $watchdog = $wdopts->{model} || 'i6300esb';
3590         push @$devices, '-device', "$watchdog$pciaddr";
3591         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3592     }
3593
3594     my $vollist = [];
3595     my $scsicontroller = {};
3596     my $ahcicontroller = {};
3597     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3598
3599     # Add iscsi initiator name if available
3600     if (my $initiator = get_initiator_name()) {
3601         push @$devices, '-iscsi', "initiator-name=$initiator";
3602     }
3603
3604     PVE::QemuConfig->foreach_volume($conf, sub {
3605         my ($ds, $drive) = @_;
3606
3607         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3608             push @$vollist, $drive->{file};
3609         }
3610
3611         # ignore efidisk here, already added in bios/fw handling code above
3612         return if $drive->{interface} eq 'efidisk';
3613
3614         $use_virtio = 1 if $ds =~ m/^virtio/;
3615
3616         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3617
3618         if ($drive->{interface} eq 'virtio'){
3619            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3620         }
3621
3622         if ($drive->{interface} eq 'scsi') {
3623
3624             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3625
3626             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3627                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3628
3629             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3630             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3631
3632             my $iothread = '';
3633             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3634                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3635                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3636             } elsif ($drive->{iothread}) {
3637                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3638             }
3639
3640             my $queues = '';
3641             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3642                 $queues = ",num_queues=$drive->{queues}";
3643             }
3644
3645             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3646                 if !$scsicontroller->{$controller};
3647             $scsicontroller->{$controller}=1;
3648         }
3649
3650         if ($drive->{interface} eq 'sata') {
3651             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3652             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3653             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3654                 if !$ahcicontroller->{$controller};
3655             $ahcicontroller->{$controller}=1;
3656         }
3657
3658         my $pbs_conf = $pbs_backing->{$ds};
3659         my $pbs_name = undef;
3660         if ($pbs_conf) {
3661             $pbs_name = "drive-$ds-pbs";
3662             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3663         }
3664
3665         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive, $pbs_name);
3666
3667         # extra protection for templates, but SATA and IDE don't support it..
3668         my $read_only = PVE::QemuConfig->is_template($conf)
3669             && $drive->{interface} ne 'sata'
3670             && $drive->{interface} ne 'ide';
3671
3672         $drive_cmd .= ',readonly=on' if $read_only;
3673
3674         push @$devices, '-drive',$drive_cmd;
3675         push @$devices, '-device', print_drivedevice_full(
3676             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3677     });
3678
3679     for (my $i = 0; $i < $MAX_NETS; $i++) {
3680         my $netname = "net$i";
3681
3682         next if !$conf->{$netname};
3683         my $d = parse_net($conf->{$netname});
3684         next if !$d;
3685
3686         $use_virtio = 1 if $d->{model} eq 'virtio';
3687
3688         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3689
3690         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3691         push @$devices, '-netdev', $netdevfull;
3692
3693         my $netdevicefull = print_netdevice_full(
3694             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3695
3696         push @$devices, '-device', $netdevicefull;
3697     }
3698
3699     if ($conf->{ivshmem}) {
3700         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3701
3702         my $bus;
3703         if ($q35) {
3704             $bus = print_pcie_addr("ivshmem");
3705         } else {
3706             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3707         }
3708
3709         my $ivshmem_name = $ivshmem->{name} // $vmid;
3710         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3711
3712         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3713         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3714             .",size=$ivshmem->{size}M";
3715     }
3716
3717     # pci.4 is nested in pci.1
3718     $bridges->{1} = 1 if $bridges->{4};
3719
3720     if (!$q35) {
3721         # add pci bridges
3722         if (min_version($machine_version, 2, 3)) {
3723            $bridges->{1} = 1;
3724            $bridges->{2} = 1;
3725         }
3726
3727         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3728
3729     }
3730
3731     for my $k (sort {$b cmp $a} keys %$bridges) {
3732         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3733
3734         my $k_name = $k;
3735         if ($k == 2 && $legacy_igd) {
3736             $k_name = "$k-igd";
3737         }
3738         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3739
3740         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3741         if ($q35) {
3742             # add after -readconfig pve-q35.cfg
3743             splice @$devices, 2, 0, '-device', $devstr;
3744         } else {
3745             unshift @$devices, '-device', $devstr if $k > 0;
3746         }
3747     }
3748
3749     if (!$kvm) {
3750         push @$machineFlags, 'accel=tcg';
3751     }
3752
3753     my $machine_type_min = $machine_type;
3754     if ($add_pve_version) {
3755         $machine_type_min =~ s/\+pve\d+$//;
3756         $machine_type_min .= "+pve$required_pve_version";
3757     }
3758     push @$machineFlags, "type=${machine_type_min}";
3759
3760     push @$cmd, @$devices;
3761     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3762     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3763     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3764
3765     if (my $vmstate = $conf->{vmstate}) {
3766         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3767         push @$vollist, $vmstate;
3768         push @$cmd, '-loadstate', $statepath;
3769         print "activating and using '$vmstate' as vmstate\n";
3770     }
3771
3772     # add custom args
3773     if ($conf->{args}) {
3774         my $aa = PVE::Tools::split_args($conf->{args});
3775         push @$cmd, @$aa;
3776     }
3777
3778     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3779 }
3780
3781 sub check_rng_source {
3782     my ($source) = @_;
3783
3784     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3785     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3786         if ! -e $source;
3787
3788     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3789     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3790         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3791         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3792         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3793             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3794             ." to the host.\n";
3795     }
3796 }
3797
3798 sub spice_port {
3799     my ($vmid) = @_;
3800
3801     my $res = mon_cmd($vmid, 'query-spice');
3802
3803     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3804 }
3805
3806 sub vm_devices_list {
3807     my ($vmid) = @_;
3808
3809     my $res = mon_cmd($vmid, 'query-pci');
3810     my $devices_to_check = [];
3811     my $devices = {};
3812     foreach my $pcibus (@$res) {
3813         push @$devices_to_check, @{$pcibus->{devices}},
3814     }
3815
3816     while (@$devices_to_check) {
3817         my $to_check = [];
3818         for my $d (@$devices_to_check) {
3819             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3820             next if !$d->{'pci_bridge'};
3821
3822             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3823             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3824         }
3825         $devices_to_check = $to_check;
3826     }
3827
3828     my $resblock = mon_cmd($vmid, 'query-block');
3829     foreach my $block (@$resblock) {
3830         if($block->{device} =~ m/^drive-(\S+)/){
3831                 $devices->{$1} = 1;
3832         }
3833     }
3834
3835     my $resmice = mon_cmd($vmid, 'query-mice');
3836     foreach my $mice (@$resmice) {
3837         if ($mice->{name} eq 'QEMU HID Tablet') {
3838             $devices->{tablet} = 1;
3839             last;
3840         }
3841     }
3842
3843     # for usb devices there is no query-usb
3844     # but we can iterate over the entries in
3845     # qom-list path=/machine/peripheral
3846     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3847     foreach my $per (@$resperipheral) {
3848         if ($per->{name} =~ m/^usb\d+$/) {
3849             $devices->{$per->{name}} = 1;
3850         }
3851     }
3852
3853     return $devices;
3854 }
3855
3856 sub vm_deviceplug {
3857     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3858
3859     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3860
3861     my $devices_list = vm_devices_list($vmid);
3862     return 1 if defined($devices_list->{$deviceid});
3863
3864     # add PCI bridge if we need it for the device
3865     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3866
3867     if ($deviceid eq 'tablet') {
3868
3869         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3870
3871     } elsif ($deviceid eq 'keyboard') {
3872
3873         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3874
3875     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3876
3877         die "usb hotplug currently not reliable\n";
3878         # since we can't reliably hot unplug all added usb devices and usb
3879         # passthrough breaks live migration we disable usb hotplugging for now
3880         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3881
3882     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3883
3884         qemu_iothread_add($vmid, $deviceid, $device);
3885
3886         qemu_driveadd($storecfg, $vmid, $device);
3887         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3888
3889         qemu_deviceadd($vmid, $devicefull);
3890         eval { qemu_deviceaddverify($vmid, $deviceid); };
3891         if (my $err = $@) {
3892             eval { qemu_drivedel($vmid, $deviceid); };
3893             warn $@ if $@;
3894             die $err;
3895         }
3896
3897     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3898
3899
3900         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3901         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3902         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3903
3904         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3905
3906         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3907             qemu_iothread_add($vmid, $deviceid, $device);
3908             $devicefull .= ",iothread=iothread-$deviceid";
3909         }
3910
3911         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3912             $devicefull .= ",num_queues=$device->{queues}";
3913         }
3914
3915         qemu_deviceadd($vmid, $devicefull);
3916         qemu_deviceaddverify($vmid, $deviceid);
3917
3918     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3919
3920         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3921         qemu_driveadd($storecfg, $vmid, $device);
3922
3923         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3924         eval { qemu_deviceadd($vmid, $devicefull); };
3925         if (my $err = $@) {
3926             eval { qemu_drivedel($vmid, $deviceid); };
3927             warn $@ if $@;
3928             die $err;
3929         }
3930
3931     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3932
3933         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3934
3935         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3936         my $use_old_bios_files = undef;
3937         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3938
3939         my $netdevicefull = print_netdevice_full(
3940             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3941         qemu_deviceadd($vmid, $netdevicefull);
3942         eval {
3943             qemu_deviceaddverify($vmid, $deviceid);
3944             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3945         };
3946         if (my $err = $@) {
3947             eval { qemu_netdevdel($vmid, $deviceid); };
3948             warn $@ if $@;
3949             die $err;
3950         }
3951
3952     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3953
3954         my $bridgeid = $2;
3955         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3956         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3957
3958         qemu_deviceadd($vmid, $devicefull);
3959         qemu_deviceaddverify($vmid, $deviceid);
3960
3961     } else {
3962         die "can't hotplug device '$deviceid'\n";
3963     }
3964
3965     return 1;
3966 }
3967
3968 # fixme: this should raise exceptions on error!
3969 sub vm_deviceunplug {
3970     my ($vmid, $conf, $deviceid) = @_;
3971
3972     my $devices_list = vm_devices_list($vmid);
3973     return 1 if !defined($devices_list->{$deviceid});
3974
3975     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3976     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3977
3978     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3979
3980         qemu_devicedel($vmid, $deviceid);
3981
3982     } elsif ($deviceid =~ m/^usb\d+$/) {
3983
3984         die "usb hotplug currently not reliable\n";
3985         # when unplugging usb devices this way, there may be remaining usb
3986         # controllers/hubs so we disable it for now
3987         #qemu_devicedel($vmid, $deviceid);
3988         #qemu_devicedelverify($vmid, $deviceid);
3989
3990     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3991
3992         qemu_devicedel($vmid, $deviceid);
3993         qemu_devicedelverify($vmid, $deviceid);
3994         qemu_drivedel($vmid, $deviceid);
3995         qemu_iothread_del($conf, $vmid, $deviceid);
3996
3997     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3998
3999         qemu_devicedel($vmid, $deviceid);
4000         qemu_devicedelverify($vmid, $deviceid);
4001         qemu_iothread_del($conf, $vmid, $deviceid);
4002
4003     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4004
4005         qemu_devicedel($vmid, $deviceid);
4006         qemu_drivedel($vmid, $deviceid);
4007         qemu_deletescsihw($conf, $vmid, $deviceid);
4008
4009     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4010
4011         qemu_devicedel($vmid, $deviceid);
4012         qemu_devicedelverify($vmid, $deviceid);
4013         qemu_netdevdel($vmid, $deviceid);
4014
4015     } else {
4016         die "can't unplug device '$deviceid'\n";
4017     }
4018
4019     return 1;
4020 }
4021
4022 sub qemu_deviceadd {
4023     my ($vmid, $devicefull) = @_;
4024
4025     $devicefull = "driver=".$devicefull;
4026     my %options =  split(/[=,]/, $devicefull);
4027
4028     mon_cmd($vmid, "device_add" , %options);
4029 }
4030
4031 sub qemu_devicedel {
4032     my ($vmid, $deviceid) = @_;
4033
4034     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4035 }
4036
4037 sub qemu_iothread_add {
4038     my($vmid, $deviceid, $device) = @_;
4039
4040     if ($device->{iothread}) {
4041         my $iothreads = vm_iothreads_list($vmid);
4042         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4043     }
4044 }
4045
4046 sub qemu_iothread_del {
4047     my($conf, $vmid, $deviceid) = @_;
4048
4049     my $confid = $deviceid;
4050     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4051         $confid = 'scsi' . $1;
4052     }
4053     my $device = parse_drive($confid, $conf->{$confid});
4054     if ($device->{iothread}) {
4055         my $iothreads = vm_iothreads_list($vmid);
4056         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4057     }
4058 }
4059
4060 sub qemu_objectadd {
4061     my($vmid, $objectid, $qomtype) = @_;
4062
4063     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4064
4065     return 1;
4066 }
4067
4068 sub qemu_objectdel {
4069     my($vmid, $objectid) = @_;
4070
4071     mon_cmd($vmid, "object-del", id => $objectid);
4072
4073     return 1;
4074 }
4075
4076 sub qemu_driveadd {
4077     my ($storecfg, $vmid, $device) = @_;
4078
4079     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4080     $drive =~ s/\\/\\\\/g;
4081     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4082
4083     # If the command succeeds qemu prints: "OK"
4084     return 1 if $ret =~ m/OK/s;
4085
4086     die "adding drive failed: $ret\n";
4087 }
4088
4089 sub qemu_drivedel {
4090     my($vmid, $deviceid) = @_;
4091
4092     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4093     $ret =~ s/^\s+//;
4094
4095     return 1 if $ret eq "";
4096
4097     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4098     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4099
4100     die "deleting drive $deviceid failed : $ret\n";
4101 }
4102
4103 sub qemu_deviceaddverify {
4104     my ($vmid, $deviceid) = @_;
4105
4106     for (my $i = 0; $i <= 5; $i++) {
4107          my $devices_list = vm_devices_list($vmid);
4108          return 1 if defined($devices_list->{$deviceid});
4109          sleep 1;
4110     }
4111
4112     die "error on hotplug device '$deviceid'\n";
4113 }
4114
4115
4116 sub qemu_devicedelverify {
4117     my ($vmid, $deviceid) = @_;
4118
4119     # need to verify that the device is correctly removed as device_del
4120     # is async and empty return is not reliable
4121
4122     for (my $i = 0; $i <= 5; $i++) {
4123          my $devices_list = vm_devices_list($vmid);
4124          return 1 if !defined($devices_list->{$deviceid});
4125          sleep 1;
4126     }
4127
4128     die "error on hot-unplugging device '$deviceid'\n";
4129 }
4130
4131 sub qemu_findorcreatescsihw {
4132     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4133
4134     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4135
4136     my $scsihwid="$controller_prefix$controller";
4137     my $devices_list = vm_devices_list($vmid);
4138
4139     if(!defined($devices_list->{$scsihwid})) {
4140         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4141     }
4142
4143     return 1;
4144 }
4145
4146 sub qemu_deletescsihw {
4147     my ($conf, $vmid, $opt) = @_;
4148
4149     my $device = parse_drive($opt, $conf->{$opt});
4150
4151     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4152         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4153         return 1;
4154     }
4155
4156     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4157
4158     my $devices_list = vm_devices_list($vmid);
4159     foreach my $opt (keys %{$devices_list}) {
4160         if (is_valid_drivename($opt)) {
4161             my $drive = parse_drive($opt, $conf->{$opt});
4162             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4163                 return 1;
4164             }
4165         }
4166     }
4167
4168     my $scsihwid="scsihw$controller";
4169
4170     vm_deviceunplug($vmid, $conf, $scsihwid);
4171
4172     return 1;
4173 }
4174
4175 sub qemu_add_pci_bridge {
4176     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4177
4178     my $bridges = {};
4179
4180     my $bridgeid;
4181
4182     print_pci_addr($device, $bridges, $arch, $machine_type);
4183
4184     while (my ($k, $v) = each %$bridges) {
4185         $bridgeid = $k;
4186     }
4187     return 1 if !defined($bridgeid) || $bridgeid < 1;
4188
4189     my $bridge = "pci.$bridgeid";
4190     my $devices_list = vm_devices_list($vmid);
4191
4192     if (!defined($devices_list->{$bridge})) {
4193         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4194     }
4195
4196     return 1;
4197 }
4198
4199 sub qemu_set_link_status {
4200     my ($vmid, $device, $up) = @_;
4201
4202     mon_cmd($vmid, "set_link", name => $device,
4203                up => $up ? JSON::true : JSON::false);
4204 }
4205
4206 sub qemu_netdevadd {
4207     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4208
4209     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4210     my %options =  split(/[=,]/, $netdev);
4211
4212     if (defined(my $vhost = $options{vhost})) {
4213         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4214     }
4215
4216     if (defined(my $queues = $options{queues})) {
4217         $options{queues} = $queues + 0;
4218     }
4219
4220     mon_cmd($vmid, "netdev_add",  %options);
4221     return 1;
4222 }
4223
4224 sub qemu_netdevdel {
4225     my ($vmid, $deviceid) = @_;
4226
4227     mon_cmd($vmid, "netdev_del", id => $deviceid);
4228 }
4229
4230 sub qemu_usb_hotplug {
4231     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4232
4233     return if !$device;
4234
4235     # remove the old one first
4236     vm_deviceunplug($vmid, $conf, $deviceid);
4237
4238     # check if xhci controller is necessary and available
4239     if ($device->{usb3}) {
4240
4241         my $devicelist = vm_devices_list($vmid);
4242
4243         if (!$devicelist->{xhci}) {
4244             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4245             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4246         }
4247     }
4248     my $d = parse_usb_device($device->{host});
4249     $d->{usb3} = $device->{usb3};
4250
4251     # add the new one
4252     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4253 }
4254
4255 sub qemu_cpu_hotplug {
4256     my ($vmid, $conf, $vcpus) = @_;
4257
4258     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4259
4260     my $sockets = 1;
4261     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4262     $sockets = $conf->{sockets} if  $conf->{sockets};
4263     my $cores = $conf->{cores} || 1;
4264     my $maxcpus = $sockets * $cores;
4265
4266     $vcpus = $maxcpus if !$vcpus;
4267
4268     die "you can't add more vcpus than maxcpus\n"
4269         if $vcpus > $maxcpus;
4270
4271     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4272
4273     if ($vcpus < $currentvcpus) {
4274
4275         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4276
4277             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4278                 qemu_devicedel($vmid, "cpu$i");
4279                 my $retry = 0;
4280                 my $currentrunningvcpus = undef;
4281                 while (1) {
4282                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4283                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4284                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4285                     $retry++;
4286                     sleep 1;
4287                 }
4288                 #update conf after each succesfull cpu unplug
4289                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4290                 PVE::QemuConfig->write_config($vmid, $conf);
4291             }
4292         } else {
4293             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4294         }
4295
4296         return;
4297     }
4298
4299     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4300     die "vcpus in running vm does not match its configuration\n"
4301         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4302
4303     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4304
4305         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4306             my $cpustr = print_cpu_device($conf, $i);
4307             qemu_deviceadd($vmid, $cpustr);
4308
4309             my $retry = 0;
4310             my $currentrunningvcpus = undef;
4311             while (1) {
4312                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4313                 last if scalar(@{$currentrunningvcpus}) == $i;
4314                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4315                 sleep 1;
4316                 $retry++;
4317             }
4318             #update conf after each succesfull cpu hotplug
4319             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4320             PVE::QemuConfig->write_config($vmid, $conf);
4321         }
4322     } else {
4323
4324         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4325             mon_cmd($vmid, "cpu-add", id => int($i));
4326         }
4327     }
4328 }
4329
4330 sub qemu_block_set_io_throttle {
4331     my ($vmid, $deviceid,
4332         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4333         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4334         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4335         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4336
4337     return if !check_running($vmid) ;
4338
4339     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4340         bps => int($bps),
4341         bps_rd => int($bps_rd),
4342         bps_wr => int($bps_wr),
4343         iops => int($iops),
4344         iops_rd => int($iops_rd),
4345         iops_wr => int($iops_wr),
4346         bps_max => int($bps_max),
4347         bps_rd_max => int($bps_rd_max),
4348         bps_wr_max => int($bps_wr_max),
4349         iops_max => int($iops_max),
4350         iops_rd_max => int($iops_rd_max),
4351         iops_wr_max => int($iops_wr_max),
4352         bps_max_length => int($bps_max_length),
4353         bps_rd_max_length => int($bps_rd_max_length),
4354         bps_wr_max_length => int($bps_wr_max_length),
4355         iops_max_length => int($iops_max_length),
4356         iops_rd_max_length => int($iops_rd_max_length),
4357         iops_wr_max_length => int($iops_wr_max_length),
4358     );
4359
4360 }
4361
4362 sub qemu_block_resize {
4363     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4364
4365     my $running = check_running($vmid);
4366
4367     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4368
4369     return if !$running;
4370
4371     my $padding = (1024 - $size % 1024) % 1024;
4372     $size = $size + $padding;
4373
4374     mon_cmd(
4375         $vmid,
4376         "block_resize",
4377         device => $deviceid,
4378         size => int($size),
4379         timeout => 60,
4380     );
4381 }
4382
4383 sub qemu_volume_snapshot {
4384     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4385
4386     my $running = check_running($vmid);
4387
4388     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4389         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4390     } else {
4391         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4392     }
4393 }
4394
4395 sub qemu_volume_snapshot_delete {
4396     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4397
4398     my $running = check_running($vmid);
4399
4400     if($running) {
4401
4402         $running = undef;
4403         my $conf = PVE::QemuConfig->load_config($vmid);
4404         PVE::QemuConfig->foreach_volume($conf, sub {
4405             my ($ds, $drive) = @_;
4406             $running = 1 if $drive->{file} eq $volid;
4407         });
4408     }
4409
4410     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4411         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4412     } else {
4413         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4414     }
4415 }
4416
4417 sub set_migration_caps {
4418     my ($vmid, $savevm) = @_;
4419
4420     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4421
4422     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4423     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4424
4425     my $cap_ref = [];
4426
4427     my $enabled_cap = {
4428         "auto-converge" => 1,
4429         "xbzrle" => 1,
4430         "x-rdma-pin-all" => 0,
4431         "zero-blocks" => 0,
4432         "compress" => 0,
4433         "dirty-bitmaps" => $dirty_bitmaps,
4434     };
4435
4436     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4437
4438     for my $supported_capability (@$supported_capabilities) {
4439         push @$cap_ref, {
4440             capability => $supported_capability->{capability},
4441             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4442         };
4443     }
4444
4445     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4446 }
4447
4448 sub foreach_volid {
4449     my ($conf, $func, @param) = @_;
4450
4451     my $volhash = {};
4452
4453     my $test_volid = sub {
4454         my ($key, $drive, $snapname) = @_;
4455
4456         my $volid = $drive->{file};
4457         return if !$volid;
4458
4459         $volhash->{$volid}->{cdrom} //= 1;
4460         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4461
4462         my $replicate = $drive->{replicate} // 1;
4463         $volhash->{$volid}->{replicate} //= 0;
4464         $volhash->{$volid}->{replicate} = 1 if $replicate;
4465
4466         $volhash->{$volid}->{shared} //= 0;
4467         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4468
4469         $volhash->{$volid}->{referenced_in_config} //= 0;
4470         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4471
4472         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4473             if defined($snapname);
4474
4475         my $size = $drive->{size};
4476         $volhash->{$volid}->{size} //= $size if $size;
4477
4478         $volhash->{$volid}->{is_vmstate} //= 0;
4479         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4480
4481         $volhash->{$volid}->{is_unused} //= 0;
4482         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4483
4484         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4485     };
4486
4487     my $include_opts = {
4488         extra_keys => ['vmstate'],
4489         include_unused => 1,
4490     };
4491
4492     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4493     foreach my $snapname (keys %{$conf->{snapshots}}) {
4494         my $snap = $conf->{snapshots}->{$snapname};
4495         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4496     }
4497
4498     foreach my $volid (keys %$volhash) {
4499         &$func($volid, $volhash->{$volid}, @param);
4500     }
4501 }
4502
4503 my $fast_plug_option = {
4504     'lock' => 1,
4505     'name' => 1,
4506     'onboot' => 1,
4507     'shares' => 1,
4508     'startup' => 1,
4509     'description' => 1,
4510     'protection' => 1,
4511     'vmstatestorage' => 1,
4512     'hookscript' => 1,
4513     'tags' => 1,
4514 };
4515
4516 # hotplug changes in [PENDING]
4517 # $selection hash can be used to only apply specified options, for
4518 # example: { cores => 1 } (only apply changed 'cores')
4519 # $errors ref is used to return error messages
4520 sub vmconfig_hotplug_pending {
4521     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4522
4523     my $defaults = load_defaults();
4524     my $arch = get_vm_arch($conf);
4525     my $machine_type = get_vm_machine($conf, undef, $arch);
4526
4527     # commit values which do not have any impact on running VM first
4528     # Note: those option cannot raise errors, we we do not care about
4529     # $selection and always apply them.
4530
4531     my $add_error = sub {
4532         my ($opt, $msg) = @_;
4533         $errors->{$opt} = "hotplug problem - $msg";
4534     };
4535
4536     my $changes = 0;
4537     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4538         if ($fast_plug_option->{$opt}) {
4539             $conf->{$opt} = $conf->{pending}->{$opt};
4540             delete $conf->{pending}->{$opt};
4541             $changes = 1;
4542         }
4543     }
4544
4545     if ($changes) {
4546         PVE::QemuConfig->write_config($vmid, $conf);
4547     }
4548
4549     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4550
4551     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4552     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4553     foreach my $opt (sort keys %$pending_delete_hash) {
4554         next if $selection && !$selection->{$opt};
4555         my $force = $pending_delete_hash->{$opt}->{force};
4556         eval {
4557             if ($opt eq 'hotplug') {
4558                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4559             } elsif ($opt eq 'tablet') {
4560                 die "skip\n" if !$hotplug_features->{usb};
4561                 if ($defaults->{tablet}) {
4562                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4563                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4564                         if $arch eq 'aarch64';
4565                 } else {
4566                     vm_deviceunplug($vmid, $conf, 'tablet');
4567                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4568                 }
4569             } elsif ($opt =~ m/^usb\d+/) {
4570                 die "skip\n";
4571                 # since we cannot reliably hot unplug usb devices we are disabling it
4572                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4573                 #vm_deviceunplug($vmid, $conf, $opt);
4574             } elsif ($opt eq 'vcpus') {
4575                 die "skip\n" if !$hotplug_features->{cpu};
4576                 qemu_cpu_hotplug($vmid, $conf, undef);
4577             } elsif ($opt eq 'balloon') {
4578                 # enable balloon device is not hotpluggable
4579                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4580                 # here we reset the ballooning value to memory
4581                 my $balloon = $conf->{memory} || $defaults->{memory};
4582                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4583             } elsif ($fast_plug_option->{$opt}) {
4584                 # do nothing
4585             } elsif ($opt =~ m/^net(\d+)$/) {
4586                 die "skip\n" if !$hotplug_features->{network};
4587                 vm_deviceunplug($vmid, $conf, $opt);
4588             } elsif (is_valid_drivename($opt)) {
4589                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4590                 vm_deviceunplug($vmid, $conf, $opt);
4591                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4592             } elsif ($opt =~ m/^memory$/) {
4593                 die "skip\n" if !$hotplug_features->{memory};
4594                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4595             } elsif ($opt eq 'cpuunits') {
4596                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4597             } elsif ($opt eq 'cpulimit') {
4598                 $cgroup->change_cpu_quota(-1, 100000);
4599             } else {
4600                 die "skip\n";
4601             }
4602         };
4603         if (my $err = $@) {
4604             &$add_error($opt, $err) if $err ne "skip\n";
4605         } else {
4606             delete $conf->{$opt};
4607             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4608         }
4609     }
4610
4611     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4612     $apply_pending_cloudinit = sub {
4613         return if $apply_pending_cloudinit_done; # once is enough
4614         $apply_pending_cloudinit_done = 1; # once is enough
4615
4616         my ($key, $value) = @_;
4617
4618         my @cloudinit_opts = keys %$confdesc_cloudinit;
4619         foreach my $opt (keys %{$conf->{pending}}) {
4620             next if !grep { $_ eq $opt } @cloudinit_opts;
4621             $conf->{$opt} = delete $conf->{pending}->{$opt};
4622         }
4623
4624         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4625         foreach my $opt (sort keys %$pending_delete_hash) {
4626             next if !grep { $_ eq $opt } @cloudinit_opts;
4627             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4628             delete $conf->{$opt};
4629         }
4630
4631         my $new_conf = { %$conf };
4632         $new_conf->{$key} = $value;
4633         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4634     };
4635
4636     foreach my $opt (keys %{$conf->{pending}}) {
4637         next if $selection && !$selection->{$opt};
4638         my $value = $conf->{pending}->{$opt};
4639         eval {
4640             if ($opt eq 'hotplug') {
4641                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4642             } elsif ($opt eq 'tablet') {
4643                 die "skip\n" if !$hotplug_features->{usb};
4644                 if ($value == 1) {
4645                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4646                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4647                         if $arch eq 'aarch64';
4648                 } elsif ($value == 0) {
4649                     vm_deviceunplug($vmid, $conf, 'tablet');
4650                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4651                 }
4652             } elsif ($opt =~ m/^usb\d+$/) {
4653                 die "skip\n";
4654                 # since we cannot reliably hot unplug usb devices we disable it for now
4655                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4656                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4657                 #die "skip\n" if !$d;
4658                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4659             } elsif ($opt eq 'vcpus') {
4660                 die "skip\n" if !$hotplug_features->{cpu};
4661                 qemu_cpu_hotplug($vmid, $conf, $value);
4662             } elsif ($opt eq 'balloon') {
4663                 # enable/disable balloning device is not hotpluggable
4664                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4665                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4666                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4667
4668                 # allow manual ballooning if shares is set to zero
4669                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4670                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4671                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4672                 }
4673             } elsif ($opt =~ m/^net(\d+)$/) {
4674                 # some changes can be done without hotplug
4675                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4676                                     $vmid, $opt, $value, $arch, $machine_type);
4677             } elsif (is_valid_drivename($opt)) {
4678                 die "skip\n" if $opt eq 'efidisk0';
4679                 # some changes can be done without hotplug
4680                 my $drive = parse_drive($opt, $value);
4681                 if (drive_is_cloudinit($drive)) {
4682                     &$apply_pending_cloudinit($opt, $value);
4683                 }
4684                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4685                                      $vmid, $opt, $value, $arch, $machine_type);
4686             } elsif ($opt =~ m/^memory$/) { #dimms
4687                 die "skip\n" if !$hotplug_features->{memory};
4688                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4689             } elsif ($opt eq 'cpuunits') {
4690                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4691             } elsif ($opt eq 'cpulimit') {
4692                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4693                 $cgroup->change_cpu_quota($cpulimit, 100000);
4694             } else {
4695                 die "skip\n";  # skip non-hot-pluggable options
4696             }
4697         };
4698         if (my $err = $@) {
4699             &$add_error($opt, $err) if $err ne "skip\n";
4700         } else {
4701             $conf->{$opt} = $value;
4702             delete $conf->{pending}->{$opt};
4703         }
4704     }
4705
4706     PVE::QemuConfig->write_config($vmid, $conf);
4707 }
4708
4709 sub try_deallocate_drive {
4710     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4711
4712     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4713         my $volid = $drive->{file};
4714         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4715             my $sid = PVE::Storage::parse_volume_id($volid);
4716             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4717
4718             # check if the disk is really unused
4719             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4720                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4721             PVE::Storage::vdisk_free($storecfg, $volid);
4722             return 1;
4723         } else {
4724             # If vm is not owner of this disk remove from config
4725             return 1;
4726         }
4727     }
4728
4729     return;
4730 }
4731
4732 sub vmconfig_delete_or_detach_drive {
4733     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4734
4735     my $drive = parse_drive($opt, $conf->{$opt});
4736
4737     my $rpcenv = PVE::RPCEnvironment::get();
4738     my $authuser = $rpcenv->get_user();
4739
4740     if ($force) {
4741         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4742         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4743     } else {
4744         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4745     }
4746 }
4747
4748
4749
4750 sub vmconfig_apply_pending {
4751     my ($vmid, $conf, $storecfg, $errors) = @_;
4752
4753     my $add_apply_error = sub {
4754         my ($opt, $msg) = @_;
4755         my $err_msg = "unable to apply pending change $opt : $msg";
4756         $errors->{$opt} = $err_msg;
4757         warn $err_msg;
4758     };
4759
4760     # cold plug
4761
4762     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4763     foreach my $opt (sort keys %$pending_delete_hash) {
4764         my $force = $pending_delete_hash->{$opt}->{force};
4765         eval {
4766             if ($opt =~ m/^unused/) {
4767                 die "internal error";
4768             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4769                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4770             }
4771         };
4772         if (my $err = $@) {
4773             $add_apply_error->($opt, $err);
4774         } else {
4775             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4776             delete $conf->{$opt};
4777         }
4778     }
4779
4780     PVE::QemuConfig->cleanup_pending($conf);
4781
4782     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4783         next if $opt eq 'delete'; # just to be sure
4784         eval {
4785             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4786                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4787             }
4788         };
4789         if (my $err = $@) {
4790             $add_apply_error->($opt, $err);
4791         } else {
4792             $conf->{$opt} = delete $conf->{pending}->{$opt};
4793         }
4794     }
4795
4796     # write all changes at once to avoid unnecessary i/o
4797     PVE::QemuConfig->write_config($vmid, $conf);
4798 }
4799
4800 sub vmconfig_update_net {
4801     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4802
4803     my $newnet = parse_net($value);
4804
4805     if ($conf->{$opt}) {
4806         my $oldnet = parse_net($conf->{$opt});
4807
4808         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4809             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4810             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4811             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4812
4813             # for non online change, we try to hot-unplug
4814             die "skip\n" if !$hotplug;
4815             vm_deviceunplug($vmid, $conf, $opt);
4816         } else {
4817
4818             die "internal error" if $opt !~ m/net(\d+)/;
4819             my $iface = "tap${vmid}i$1";
4820
4821             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4822                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4823                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4824                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4825                 PVE::Network::tap_unplug($iface);
4826
4827                 if ($have_sdn) {
4828                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4829                 } else {
4830                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4831                 }
4832             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4833                 # Rate can be applied on its own but any change above needs to
4834                 # include the rate in tap_plug since OVS resets everything.
4835                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4836             }
4837
4838             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4839                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4840             }
4841
4842             return 1;
4843         }
4844     }
4845
4846     if ($hotplug) {
4847         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4848     } else {
4849         die "skip\n";
4850     }
4851 }
4852
4853 sub vmconfig_update_disk {
4854     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4855
4856     my $drive = parse_drive($opt, $value);
4857
4858     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4859         my $media = $drive->{media} || 'disk';
4860         my $oldmedia = $old_drive->{media} || 'disk';
4861         die "unable to change media type\n" if $media ne $oldmedia;
4862
4863         if (!drive_is_cdrom($old_drive)) {
4864
4865             if ($drive->{file} ne $old_drive->{file}) {
4866
4867                 die "skip\n" if !$hotplug;
4868
4869                 # unplug and register as unused
4870                 vm_deviceunplug($vmid, $conf, $opt);
4871                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4872
4873             } else {
4874                 # update existing disk
4875
4876                 # skip non hotpluggable value
4877                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4878                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4879                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4880                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4881                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4882                     die "skip\n";
4883                 }
4884
4885                 # apply throttle
4886                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4887                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4888                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4889                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4890                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4891                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4892                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4893                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4894                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4895                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4896                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4897                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4898                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4899                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4900                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4901                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4902                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4903                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4904
4905                     qemu_block_set_io_throttle(
4906                         $vmid,"drive-$opt",
4907                         ($drive->{mbps} || 0)*1024*1024,
4908                         ($drive->{mbps_rd} || 0)*1024*1024,
4909                         ($drive->{mbps_wr} || 0)*1024*1024,
4910                          $drive->{iops} || 0,
4911                          $drive->{iops_rd} || 0,
4912                          $drive->{iops_wr} || 0,
4913                         ($drive->{mbps_max} || 0)*1024*1024,
4914                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4915                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4916                          $drive->{iops_max} || 0,
4917                          $drive->{iops_rd_max} || 0,
4918                          $drive->{iops_wr_max} || 0,
4919                          $drive->{bps_max_length} || 1,
4920                          $drive->{bps_rd_max_length} || 1,
4921                          $drive->{bps_wr_max_length} || 1,
4922                          $drive->{iops_max_length} || 1,
4923                          $drive->{iops_rd_max_length} || 1,
4924                          $drive->{iops_wr_max_length} || 1,
4925                     );
4926
4927                 }
4928
4929                 return 1;
4930             }
4931
4932         } else { # cdrom
4933
4934             if ($drive->{file} eq 'none') {
4935                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4936                 if (drive_is_cloudinit($old_drive)) {
4937                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4938                 }
4939             } else {
4940                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4941
4942                 # force eject if locked
4943                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4944
4945                 if ($path) {
4946                     mon_cmd($vmid, "blockdev-change-medium",
4947                         id => "$opt", filename => "$path");
4948                 }
4949             }
4950
4951             return 1;
4952         }
4953     }
4954
4955     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4956     # hotplug new disks
4957     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4958     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4959 }
4960
4961 # called in locked context by incoming migration
4962 sub vm_migrate_get_nbd_disks {
4963     my ($storecfg, $conf, $replicated_volumes) = @_;
4964
4965     my $local_volumes = {};
4966     PVE::QemuConfig->foreach_volume($conf, sub {
4967         my ($ds, $drive) = @_;
4968
4969         return if drive_is_cdrom($drive);
4970
4971         my $volid = $drive->{file};
4972
4973         return if !$volid;
4974
4975         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4976
4977         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4978         return if $scfg->{shared};
4979
4980         # replicated disks re-use existing state via bitmap
4981         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4982         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4983     });
4984     return $local_volumes;
4985 }
4986
4987 # called in locked context by incoming migration
4988 sub vm_migrate_alloc_nbd_disks {
4989     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4990
4991     my $format = undef;
4992
4993     my $nbd = {};
4994     foreach my $opt (sort keys %$source_volumes) {
4995         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4996
4997         if ($use_existing) {
4998             $nbd->{$opt}->{drivestr} = print_drive($drive);
4999             $nbd->{$opt}->{volid} = $volid;
5000             $nbd->{$opt}->{replicated} = 1;
5001             next;
5002         }
5003
5004         # If a remote storage is specified and the format of the original
5005         # volume is not available there, fall back to the default format.
5006         # Otherwise use the same format as the original.
5007         if (!$storagemap->{identity}) {
5008             $storeid = map_storage($storagemap, $storeid);
5009             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5010             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5011             my $fileFormat = qemu_img_format($scfg, $volname);
5012             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5013         } else {
5014             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5015             $format = qemu_img_format($scfg, $volname);
5016         }
5017
5018         my $size = $drive->{size} / 1024;
5019         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5020         my $newdrive = $drive;
5021         $newdrive->{format} = $format;
5022         $newdrive->{file} = $newvolid;
5023         my $drivestr = print_drive($newdrive);
5024         $nbd->{$opt}->{drivestr} = $drivestr;
5025         $nbd->{$opt}->{volid} = $newvolid;
5026     }
5027
5028     return $nbd;
5029 }
5030
5031 # see vm_start_nolock for parameters, additionally:
5032 # migrate_opts:
5033 #   storagemap = parsed storage map for allocating NBD disks
5034 sub vm_start {
5035     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5036
5037     return PVE::QemuConfig->lock_config($vmid, sub {
5038         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5039
5040         die "you can't start a vm if it's a template\n"
5041             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5042
5043         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5044         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5045
5046         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5047
5048         if ($has_backup_lock && $running) {
5049             # a backup is currently running, attempt to start the guest in the
5050             # existing QEMU instance
5051             return vm_resume($vmid);
5052         }
5053
5054         PVE::QemuConfig->check_lock($conf)
5055             if !($params->{skiplock} || $has_suspended_lock);
5056
5057         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5058
5059         die "VM $vmid already running\n" if $running;
5060
5061         if (my $storagemap = $migrate_opts->{storagemap}) {
5062             my $replicated = $migrate_opts->{replicated_volumes};
5063             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5064             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5065
5066             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5067                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5068             }
5069         }
5070
5071         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5072     });
5073 }
5074
5075
5076 # params:
5077 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5078 #   skiplock => 0/1, skip checking for config lock
5079 #   skiptemplate => 0/1, skip checking whether VM is template
5080 #   forcemachine => to force Qemu machine (rollback/migration)
5081 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5082 #   timeout => in seconds
5083 #   paused => start VM in paused state (backup)
5084 #   resume => resume from hibernation
5085 #   pbs-backing => {
5086 #      sata0 => {
5087 #         repository
5088 #         snapshot
5089 #         keyfile
5090 #         archive
5091 #      },
5092 #      virtio2 => ...
5093 #   }
5094 # migrate_opts:
5095 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5096 #   migratedfrom => source node
5097 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5098 #   network => CIDR of migration network
5099 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5100 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5101 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5102 sub vm_start_nolock {
5103     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5104
5105     my $statefile = $params->{statefile};
5106     my $resume = $params->{resume};
5107
5108     my $migratedfrom = $migrate_opts->{migratedfrom};
5109     my $migration_type = $migrate_opts->{type};
5110
5111     my $res = {};
5112
5113     # clean up leftover reboot request files
5114     eval { clear_reboot_request($vmid); };
5115     warn $@ if $@;
5116
5117     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5118         vmconfig_apply_pending($vmid, $conf, $storecfg);
5119         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5120     }
5121
5122     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5123
5124     my $defaults = load_defaults();
5125
5126     # set environment variable useful inside network script
5127     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5128
5129     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5130
5131     my $forcemachine = $params->{forcemachine};
5132     my $forcecpu = $params->{forcecpu};
5133     if ($resume) {
5134         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5135         $forcemachine = $conf->{runningmachine};
5136         $forcecpu = $conf->{runningcpu};
5137         print "Resuming suspended VM\n";
5138     }
5139
5140     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5141         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5142
5143     my $migration_ip;
5144     my $get_migration_ip = sub {
5145         my ($nodename) = @_;
5146
5147         return $migration_ip if defined($migration_ip);
5148
5149         my $cidr = $migrate_opts->{network};
5150
5151         if (!defined($cidr)) {
5152             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5153             $cidr = $dc_conf->{migration}->{network};
5154         }
5155
5156         if (defined($cidr)) {
5157             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5158
5159             die "could not get IP: no address configured on local " .
5160                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5161
5162             die "could not get IP: multiple addresses configured on local " .
5163                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5164
5165             $migration_ip = @$ips[0];
5166         }
5167
5168         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5169             if !defined($migration_ip);
5170
5171         return $migration_ip;
5172     };
5173
5174     my $migrate_uri;
5175     if ($statefile) {
5176         if ($statefile eq 'tcp') {
5177             my $localip = "localhost";
5178             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5179             my $nodename = nodename();
5180
5181             if (!defined($migration_type)) {
5182                 if (defined($datacenterconf->{migration}->{type})) {
5183                     $migration_type = $datacenterconf->{migration}->{type};
5184                 } else {
5185                     $migration_type = 'secure';
5186                 }
5187             }
5188
5189             if ($migration_type eq 'insecure') {
5190                 $localip = $get_migration_ip->($nodename);
5191                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5192             }
5193
5194             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5195             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5196             $migrate_uri = "tcp:${localip}:${migrate_port}";
5197             push @$cmd, '-incoming', $migrate_uri;
5198             push @$cmd, '-S';
5199
5200         } elsif ($statefile eq 'unix') {
5201             # should be default for secure migrations as a ssh TCP forward
5202             # tunnel is not deterministic reliable ready and fails regurarly
5203             # to set up in time, so use UNIX socket forwards
5204             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5205             unlink $socket_addr;
5206
5207             $migrate_uri = "unix:$socket_addr";
5208
5209             push @$cmd, '-incoming', $migrate_uri;
5210             push @$cmd, '-S';
5211
5212         } elsif (-e $statefile) {
5213             push @$cmd, '-loadstate', $statefile;
5214         } else {
5215             my $statepath = PVE::Storage::path($storecfg, $statefile);
5216             push @$vollist, $statefile;
5217             push @$cmd, '-loadstate', $statepath;
5218         }
5219     } elsif ($params->{paused}) {
5220         push @$cmd, '-S';
5221     }
5222
5223     # host pci devices
5224     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5225       my $d = parse_hostpci($conf->{"hostpci$i"});
5226       next if !$d;
5227       my $pcidevices = $d->{pciid};
5228       foreach my $pcidevice (@$pcidevices) {
5229             my $pciid = $pcidevice->{id};
5230
5231             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5232             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5233             die "no pci device info for device '$pciid'\n" if !$info;
5234
5235             if ($d->{mdev}) {
5236                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5237                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5238             } else {
5239                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5240                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5241                 die "can't reset PCI device '$pciid'\n"
5242                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5243             }
5244       }
5245     }
5246
5247     PVE::Storage::activate_volumes($storecfg, $vollist);
5248
5249     eval {
5250         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5251             outfunc => sub {}, errfunc => sub {});
5252     };
5253     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5254     # timeout should be more than enough here...
5255     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5256
5257     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5258                                               : $defaults->{cpuunits};
5259
5260     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5261     my %run_params = (
5262         timeout => $statefile ? undef : $start_timeout,
5263         umask => 0077,
5264         noerr => 1,
5265     );
5266
5267     # when migrating, prefix QEMU output so other side can pick up any
5268     # errors that might occur and show the user
5269     if ($migratedfrom) {
5270         $run_params{quiet} = 1;
5271         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5272     }
5273
5274     my %properties = (
5275         Slice => 'qemu.slice',
5276         KillMode => 'none'
5277     );
5278
5279     if (PVE::CGroup::cgroup_mode() == 2) {
5280         $properties{CPUWeight} = $cpuunits;
5281     } else {
5282         $properties{CPUShares} = $cpuunits;
5283     }
5284
5285     if (my $cpulimit = $conf->{cpulimit}) {
5286         $properties{CPUQuota} = int($cpulimit * 100);
5287     }
5288     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5289
5290     my $run_qemu = sub {
5291         PVE::Tools::run_fork sub {
5292             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5293
5294             my $exitcode = run_command($cmd, %run_params);
5295             die "QEMU exited with code $exitcode\n" if $exitcode;
5296         };
5297     };
5298
5299     if ($conf->{hugepages}) {
5300
5301         my $code = sub {
5302             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5303             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5304
5305             PVE::QemuServer::Memory::hugepages_mount();
5306             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5307
5308             eval { $run_qemu->() };
5309             if (my $err = $@) {
5310                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5311                     if !$conf->{keephugepages};
5312                 die $err;
5313             }
5314
5315             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5316                 if !$conf->{keephugepages};
5317         };
5318         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5319
5320     } else {
5321         eval { $run_qemu->() };
5322     }
5323
5324     if (my $err = $@) {
5325         # deactivate volumes if start fails
5326         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5327         die "start failed: $err";
5328     }
5329
5330     print "migration listens on $migrate_uri\n" if $migrate_uri;
5331     $res->{migrate_uri} = $migrate_uri;
5332
5333     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5334         eval { mon_cmd($vmid, "cont"); };
5335         warn $@ if $@;
5336     }
5337
5338     #start nbd server for storage migration
5339     if (my $nbd = $migrate_opts->{nbd}) {
5340         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5341
5342         my $migrate_storage_uri;
5343         # nbd_protocol_version > 0 for unix socket support
5344         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5345             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5346             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5347             $migrate_storage_uri = "nbd:unix:$socket_path";
5348         } else {
5349             my $nodename = nodename();
5350             my $localip = $get_migration_ip->($nodename);
5351             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5352             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5353
5354             mon_cmd($vmid, "nbd-server-start", addr => {
5355                 type => 'inet',
5356                 data => {
5357                     host => "${localip}",
5358                     port => "${storage_migrate_port}",
5359                 },
5360             });
5361             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5362             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5363         }
5364
5365         $res->{migrate_storage_uri} = $migrate_storage_uri;
5366
5367         foreach my $opt (sort keys %$nbd) {
5368             my $drivestr = $nbd->{$opt}->{drivestr};
5369             my $volid = $nbd->{$opt}->{volid};
5370             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5371             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5372             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5373             print "re-using replicated volume: $opt - $volid\n"
5374                 if $nbd->{$opt}->{replicated};
5375
5376             $res->{drives}->{$opt} = $nbd->{$opt};
5377             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5378         }
5379     }
5380
5381     if ($migratedfrom) {
5382         eval {
5383             set_migration_caps($vmid);
5384         };
5385         warn $@ if $@;
5386
5387         if ($spice_port) {
5388             print "spice listens on port $spice_port\n";
5389             $res->{spice_port} = $spice_port;
5390             if ($migrate_opts->{spice_ticket}) {
5391                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5392                     $migrate_opts->{spice_ticket});
5393                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5394             }
5395         }
5396
5397     } else {
5398         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5399             if !$statefile && $conf->{balloon};
5400
5401         foreach my $opt (keys %$conf) {
5402             next if $opt !~  m/^net\d+$/;
5403             my $nicconf = parse_net($conf->{$opt});
5404             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5405         }
5406     }
5407
5408     mon_cmd($vmid, 'qom-set',
5409                 path => "machine/peripheral/balloon0",
5410                 property => "guest-stats-polling-interval",
5411                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5412
5413     if ($resume) {
5414         print "Resumed VM, removing state\n";
5415         if (my $vmstate = $conf->{vmstate}) {
5416             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5417             PVE::Storage::vdisk_free($storecfg, $vmstate);
5418         }
5419         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5420         PVE::QemuConfig->write_config($vmid, $conf);
5421     }
5422
5423     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5424
5425     return $res;
5426 }
5427
5428 sub vm_commandline {
5429     my ($storecfg, $vmid, $snapname) = @_;
5430
5431     my $conf = PVE::QemuConfig->load_config($vmid);
5432     my $forcemachine;
5433     my $forcecpu;
5434
5435     if ($snapname) {
5436         my $snapshot = $conf->{snapshots}->{$snapname};
5437         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5438
5439         # check for machine or CPU overrides in snapshot
5440         $forcemachine = $snapshot->{runningmachine};
5441         $forcecpu = $snapshot->{runningcpu};
5442
5443         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5444
5445         $conf = $snapshot;
5446     }
5447
5448     my $defaults = load_defaults();
5449
5450     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5451         $forcemachine, $forcecpu);
5452
5453     return PVE::Tools::cmd2string($cmd);
5454 }
5455
5456 sub vm_reset {
5457     my ($vmid, $skiplock) = @_;
5458
5459     PVE::QemuConfig->lock_config($vmid, sub {
5460
5461         my $conf = PVE::QemuConfig->load_config($vmid);
5462
5463         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5464
5465         mon_cmd($vmid, "system_reset");
5466     });
5467 }
5468
5469 sub get_vm_volumes {
5470     my ($conf) = @_;
5471
5472     my $vollist = [];
5473     foreach_volid($conf, sub {
5474         my ($volid, $attr) = @_;
5475
5476         return if $volid =~ m|^/|;
5477
5478         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5479         return if !$sid;
5480
5481         push @$vollist, $volid;
5482     });
5483
5484     return $vollist;
5485 }
5486
5487 sub vm_stop_cleanup {
5488     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5489
5490     eval {
5491
5492         if (!$keepActive) {
5493             my $vollist = get_vm_volumes($conf);
5494             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5495         }
5496
5497         foreach my $ext (qw(mon qmp pid vnc qga)) {
5498             unlink "/var/run/qemu-server/${vmid}.$ext";
5499         }
5500
5501         if ($conf->{ivshmem}) {
5502             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5503             # just delete it for now, VMs which have this already open do not
5504             # are affected, but new VMs will get a separated one. If this
5505             # becomes an issue we either add some sort of ref-counting or just
5506             # add a "don't delete on stop" flag to the ivshmem format.
5507             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5508         }
5509
5510         foreach my $key (keys %$conf) {
5511             next if $key !~ m/^hostpci(\d+)$/;
5512             my $hostpciindex = $1;
5513             my $d = parse_hostpci($conf->{$key});
5514             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5515
5516             foreach my $pci (@{$d->{pciid}}) {
5517                 my $pciid = $pci->{id};
5518                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5519             }
5520         }
5521
5522         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5523     };
5524     warn $@ if $@; # avoid errors - just warn
5525 }
5526
5527 # call only in locked context
5528 sub _do_vm_stop {
5529     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5530
5531     my $pid = check_running($vmid, $nocheck);
5532     return if !$pid;
5533
5534     my $conf;
5535     if (!$nocheck) {
5536         $conf = PVE::QemuConfig->load_config($vmid);
5537         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5538         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5539             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5540             $timeout = $opts->{down} if $opts->{down};
5541         }
5542         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5543     }
5544
5545     eval {
5546         if ($shutdown) {
5547             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5548                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5549             } else {
5550                 mon_cmd($vmid, "system_powerdown");
5551             }
5552         } else {
5553             mon_cmd($vmid, "quit");
5554         }
5555     };
5556     my $err = $@;
5557
5558     if (!$err) {
5559         $timeout = 60 if !defined($timeout);
5560
5561         my $count = 0;
5562         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5563             $count++;
5564             sleep 1;
5565         }
5566
5567         if ($count >= $timeout) {
5568             if ($force) {
5569                 warn "VM still running - terminating now with SIGTERM\n";
5570                 kill 15, $pid;
5571             } else {
5572                 die "VM quit/powerdown failed - got timeout\n";
5573             }
5574         } else {
5575             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5576             return;
5577         }
5578     } else {
5579        if (!check_running($vmid, $nocheck)) {
5580             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5581             return;
5582        }
5583        if ($force) {
5584             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5585             kill 15, $pid;
5586         } else {
5587             die "VM quit/powerdown failed\n";
5588         }
5589     }
5590
5591     # wait again
5592     $timeout = 10;
5593
5594     my $count = 0;
5595     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5596         $count++;
5597         sleep 1;
5598     }
5599
5600     if ($count >= $timeout) {
5601         warn "VM still running - terminating now with SIGKILL\n";
5602         kill 9, $pid;
5603         sleep 1;
5604     }
5605
5606     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5607 }
5608
5609 # Note: use $nocheck to skip tests if VM configuration file exists.
5610 # We need that when migration VMs to other nodes (files already moved)
5611 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5612 sub vm_stop {
5613     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5614
5615     $force = 1 if !defined($force) && !$shutdown;
5616
5617     if ($migratedfrom){
5618         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5619         kill 15, $pid if $pid;
5620         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5621         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5622         return;
5623     }
5624
5625     PVE::QemuConfig->lock_config($vmid, sub {
5626         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5627    });
5628 }
5629
5630 sub vm_reboot {
5631     my ($vmid, $timeout) = @_;
5632
5633     PVE::QemuConfig->lock_config($vmid, sub {
5634         eval {
5635
5636             # only reboot if running, as qmeventd starts it again on a stop event
5637             return if !check_running($vmid);
5638
5639             create_reboot_request($vmid);
5640
5641             my $storecfg = PVE::Storage::config();
5642             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5643
5644         };
5645         if (my $err = $@) {
5646             # avoid that the next normal shutdown will be confused for a reboot
5647             clear_reboot_request($vmid);
5648             die $err;
5649         }
5650    });
5651 }
5652
5653 # note: if using the statestorage parameter, the caller has to check privileges
5654 sub vm_suspend {
5655     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5656
5657     my $conf;
5658     my $path;
5659     my $storecfg;
5660     my $vmstate;
5661
5662     PVE::QemuConfig->lock_config($vmid, sub {
5663
5664         $conf = PVE::QemuConfig->load_config($vmid);
5665
5666         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5667         PVE::QemuConfig->check_lock($conf)
5668             if !($skiplock || $is_backing_up);
5669
5670         die "cannot suspend to disk during backup\n"
5671             if $is_backing_up && $includestate;
5672
5673         if ($includestate) {
5674             $conf->{lock} = 'suspending';
5675             my $date = strftime("%Y-%m-%d", localtime(time()));
5676             $storecfg = PVE::Storage::config();
5677             if (!$statestorage) {
5678                 $statestorage = find_vmstate_storage($conf, $storecfg);
5679                 # check permissions for the storage
5680                 my $rpcenv = PVE::RPCEnvironment::get();
5681                 if ($rpcenv->{type} ne 'cli') {
5682                     my $authuser = $rpcenv->get_user();
5683                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5684                 }
5685             }
5686
5687
5688             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5689                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5690             $path = PVE::Storage::path($storecfg, $vmstate);
5691             PVE::QemuConfig->write_config($vmid, $conf);
5692         } else {
5693             mon_cmd($vmid, "stop");
5694         }
5695     });
5696
5697     if ($includestate) {
5698         # save vm state
5699         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5700
5701         eval {
5702             set_migration_caps($vmid, 1);
5703             mon_cmd($vmid, "savevm-start", statefile => $path);
5704             for(;;) {
5705                 my $state = mon_cmd($vmid, "query-savevm");
5706                 if (!$state->{status}) {
5707                     die "savevm not active\n";
5708                 } elsif ($state->{status} eq 'active') {
5709                     sleep(1);
5710                     next;
5711                 } elsif ($state->{status} eq 'completed') {
5712                     print "State saved, quitting\n";
5713                     last;
5714                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5715                     die "query-savevm failed with error '$state->{error}'\n"
5716                 } else {
5717                     die "query-savevm returned status '$state->{status}'\n";
5718                 }
5719             }
5720         };
5721         my $err = $@;
5722
5723         PVE::QemuConfig->lock_config($vmid, sub {
5724             $conf = PVE::QemuConfig->load_config($vmid);
5725             if ($err) {
5726                 # cleanup, but leave suspending lock, to indicate something went wrong
5727                 eval {
5728                     mon_cmd($vmid, "savevm-end");
5729                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5730                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5731                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5732                     PVE::QemuConfig->write_config($vmid, $conf);
5733                 };
5734                 warn $@ if $@;
5735                 die $err;
5736             }
5737
5738             die "lock changed unexpectedly\n"
5739                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5740
5741             mon_cmd($vmid, "quit");
5742             $conf->{lock} = 'suspended';
5743             PVE::QemuConfig->write_config($vmid, $conf);
5744         });
5745     }
5746 }
5747
5748 sub vm_resume {
5749     my ($vmid, $skiplock, $nocheck) = @_;
5750
5751     PVE::QemuConfig->lock_config($vmid, sub {
5752         my $res = mon_cmd($vmid, 'query-status');
5753         my $resume_cmd = 'cont';
5754         my $reset = 0;
5755
5756         if ($res->{status}) {
5757             return if $res->{status} eq 'running'; # job done, go home
5758             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5759             $reset = 1 if $res->{status} eq 'shutdown';
5760         }
5761
5762         if (!$nocheck) {
5763
5764             my $conf = PVE::QemuConfig->load_config($vmid);
5765
5766             PVE::QemuConfig->check_lock($conf)
5767                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5768         }
5769
5770         if ($reset) {
5771             # required if a VM shuts down during a backup and we get a resume
5772             # request before the backup finishes for example
5773             mon_cmd($vmid, "system_reset");
5774         }
5775         mon_cmd($vmid, $resume_cmd);
5776     });
5777 }
5778
5779 sub vm_sendkey {
5780     my ($vmid, $skiplock, $key) = @_;
5781
5782     PVE::QemuConfig->lock_config($vmid, sub {
5783
5784         my $conf = PVE::QemuConfig->load_config($vmid);
5785
5786         # there is no qmp command, so we use the human monitor command
5787         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5788         die $res if $res ne '';
5789     });
5790 }
5791
5792 # vzdump restore implementaion
5793
5794 sub tar_archive_read_firstfile {
5795     my $archive = shift;
5796
5797     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5798
5799     # try to detect archive type first
5800     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5801         die "unable to open file '$archive'\n";
5802     my $firstfile = <$fh>;
5803     kill 15, $pid;
5804     close $fh;
5805
5806     die "ERROR: archive contaions no data\n" if !$firstfile;
5807     chomp $firstfile;
5808
5809     return $firstfile;
5810 }
5811
5812 sub tar_restore_cleanup {
5813     my ($storecfg, $statfile) = @_;
5814
5815     print STDERR "starting cleanup\n";
5816
5817     if (my $fd = IO::File->new($statfile, "r")) {
5818         while (defined(my $line = <$fd>)) {
5819             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5820                 my $volid = $2;
5821                 eval {
5822                     if ($volid =~ m|^/|) {
5823                         unlink $volid || die 'unlink failed\n';
5824                     } else {
5825                         PVE::Storage::vdisk_free($storecfg, $volid);
5826                     }
5827                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5828                 };
5829                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5830             } else {
5831                 print STDERR "unable to parse line in statfile - $line";
5832             }
5833         }
5834         $fd->close();
5835     }
5836 }
5837
5838 sub restore_file_archive {
5839     my ($archive, $vmid, $user, $opts) = @_;
5840
5841     return restore_vma_archive($archive, $vmid, $user, $opts)
5842         if $archive eq '-';
5843
5844     my $info = PVE::Storage::archive_info($archive);
5845     my $format = $opts->{format} // $info->{format};
5846     my $comp = $info->{compression};
5847
5848     # try to detect archive format
5849     if ($format eq 'tar') {
5850         return restore_tar_archive($archive, $vmid, $user, $opts);
5851     } else {
5852         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5853     }
5854 }
5855
5856 # hepler to remove disks that will not be used after restore
5857 my $restore_cleanup_oldconf = sub {
5858     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5859
5860     PVE::QemuConfig->foreach_volume($oldconf, sub {
5861         my ($ds, $drive) = @_;
5862
5863         return if drive_is_cdrom($drive, 1);
5864
5865         my $volid = $drive->{file};
5866         return if !$volid || $volid =~ m|^/|;
5867
5868         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5869         return if !$path || !$owner || ($owner != $vmid);
5870
5871         # Note: only delete disk we want to restore
5872         # other volumes will become unused
5873         if ($virtdev_hash->{$ds}) {
5874             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5875             if (my $err = $@) {
5876                 warn $err;
5877             }
5878         }
5879     });
5880
5881     # delete vmstate files, after the restore we have no snapshots anymore
5882     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5883         my $snap = $oldconf->{snapshots}->{$snapname};
5884         if ($snap->{vmstate}) {
5885             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5886             if (my $err = $@) {
5887                 warn $err;
5888             }
5889         }
5890     }
5891 };
5892
5893 # Helper to parse vzdump backup device hints
5894 #
5895 # $rpcenv: Environment, used to ckeck storage permissions
5896 # $user: User ID, to check storage permissions
5897 # $storecfg: Storage configuration
5898 # $fh: the file handle for reading the configuration
5899 # $devinfo: should contain device sizes for all backu-up'ed devices
5900 # $options: backup options (pool, default storage)
5901 #
5902 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5903 my $parse_backup_hints = sub {
5904     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5905
5906     my $virtdev_hash = {};
5907
5908     while (defined(my $line = <$fh>)) {
5909         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5910             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5911             die "archive does not contain data for drive '$virtdev'\n"
5912                 if !$devinfo->{$devname};
5913
5914             if (defined($options->{storage})) {
5915                 $storeid = $options->{storage} || 'local';
5916             } elsif (!$storeid) {
5917                 $storeid = 'local';
5918             }
5919             $format = 'raw' if !$format;
5920             $devinfo->{$devname}->{devname} = $devname;
5921             $devinfo->{$devname}->{virtdev} = $virtdev;
5922             $devinfo->{$devname}->{format} = $format;
5923             $devinfo->{$devname}->{storeid} = $storeid;
5924
5925             # check permission on storage
5926             my $pool = $options->{pool}; # todo: do we need that?
5927             if ($user ne 'root@pam') {
5928                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5929             }
5930
5931             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5932         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5933             my $virtdev = $1;
5934             my $drive = parse_drive($virtdev, $2);
5935             if (drive_is_cloudinit($drive)) {
5936                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5937                 $storeid = $options->{storage} if defined ($options->{storage});
5938                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5939                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5940
5941                 $virtdev_hash->{$virtdev} = {
5942                     format => $format,
5943                     storeid => $storeid,
5944                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5945                     is_cloudinit => 1,
5946                 };
5947             }
5948         }
5949     }
5950
5951     return $virtdev_hash;
5952 };
5953
5954 # Helper to allocate and activate all volumes required for a restore
5955 #
5956 # $storecfg: Storage configuration
5957 # $virtdev_hash: as returned by parse_backup_hints()
5958 #
5959 # Returns: { $virtdev => $volid }
5960 my $restore_allocate_devices = sub {
5961     my ($storecfg, $virtdev_hash, $vmid) = @_;
5962
5963     my $map = {};
5964     foreach my $virtdev (sort keys %$virtdev_hash) {
5965         my $d = $virtdev_hash->{$virtdev};
5966         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5967         my $storeid = $d->{storeid};
5968         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5969
5970         # test if requested format is supported
5971         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5972         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5973         $d->{format} = $defFormat if !$supported;
5974
5975         my $name;
5976         if ($d->{is_cloudinit}) {
5977             $name = "vm-$vmid-cloudinit";
5978             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5979             if ($scfg->{path}) {
5980                 $name .= ".$d->{format}";
5981             }
5982         }
5983
5984         my $volid = PVE::Storage::vdisk_alloc(
5985             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5986
5987         print STDERR "new volume ID is '$volid'\n";
5988         $d->{volid} = $volid;
5989
5990         PVE::Storage::activate_volumes($storecfg, [$volid]);
5991
5992         $map->{$virtdev} = $volid;
5993     }
5994
5995     return $map;
5996 };
5997
5998 sub restore_update_config_line {
5999     my ($cookie, $map, $line, $unique) = @_;
6000
6001     return '' if $line =~ m/^\#qmdump\#/;
6002     return '' if $line =~ m/^\#vzdump\#/;
6003     return '' if $line =~ m/^lock:/;
6004     return '' if $line =~ m/^unused\d+:/;
6005     return '' if $line =~ m/^parent:/;
6006
6007     my $res = '';
6008
6009     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6010     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6011         # try to convert old 1.X settings
6012         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6013         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6014             my ($model, $macaddr) = split(/\=/, $devconfig);
6015             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6016             my $net = {
6017                 model => $model,
6018                 bridge => "vmbr$ind",
6019                 macaddr => $macaddr,
6020             };
6021             my $netstr = print_net($net);
6022
6023             $res .= "net$cookie->{netcount}: $netstr\n";
6024             $cookie->{netcount}++;
6025         }
6026     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6027         my ($id, $netstr) = ($1, $2);
6028         my $net = parse_net($netstr);
6029         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6030         $netstr = print_net($net);
6031         $res .= "$id: $netstr\n";
6032     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6033         my $virtdev = $1;
6034         my $value = $3;
6035         my $di = parse_drive($virtdev, $value);
6036         if (defined($di->{backup}) && !$di->{backup}) {
6037             $res .= "#$line";
6038         } elsif ($map->{$virtdev}) {
6039             delete $di->{format}; # format can change on restore
6040             $di->{file} = $map->{$virtdev};
6041             $value = print_drive($di);
6042             $res .= "$virtdev: $value\n";
6043         } else {
6044             $res .= $line;
6045         }
6046     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6047         my $vmgenid = $1;
6048         if ($vmgenid ne '0') {
6049             # always generate a new vmgenid if there was a valid one setup
6050             $vmgenid = generate_uuid();
6051         }
6052         $res .= "vmgenid: $vmgenid\n";
6053     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6054         my ($uuid, $uuid_str);
6055         UUID::generate($uuid);
6056         UUID::unparse($uuid, $uuid_str);
6057         my $smbios1 = parse_smbios1($2);
6058         $smbios1->{uuid} = $uuid_str;
6059         $res .= $1.print_smbios1($smbios1)."\n";
6060     } else {
6061         $res .= $line;
6062     }
6063
6064     return $res;
6065 }
6066
6067 my $restore_deactivate_volumes = sub {
6068     my ($storecfg, $devinfo) = @_;
6069
6070     my $vollist = [];
6071     foreach my $devname (keys %$devinfo) {
6072         my $volid = $devinfo->{$devname}->{volid};
6073         push @$vollist, $volid if $volid;
6074     }
6075
6076     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6077 };
6078
6079 my $restore_destroy_volumes = sub {
6080     my ($storecfg, $devinfo) = @_;
6081
6082     foreach my $devname (keys %$devinfo) {
6083         my $volid = $devinfo->{$devname}->{volid};
6084         next if !$volid;
6085         eval {
6086             if ($volid =~ m|^/|) {
6087                 unlink $volid || die 'unlink failed\n';
6088             } else {
6089                 PVE::Storage::vdisk_free($storecfg, $volid);
6090             }
6091             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6092         };
6093         print STDERR "unable to cleanup '$volid' - $@" if $@;
6094     }
6095 };
6096
6097 sub scan_volids {
6098     my ($cfg, $vmid) = @_;
6099
6100     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6101
6102     my $volid_hash = {};
6103     foreach my $storeid (keys %$info) {
6104         foreach my $item (@{$info->{$storeid}}) {
6105             next if !($item->{volid} && $item->{size});
6106             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6107             $volid_hash->{$item->{volid}} = $item;
6108         }
6109     }
6110
6111     return $volid_hash;
6112 }
6113
6114 sub update_disk_config {
6115     my ($vmid, $conf, $volid_hash) = @_;
6116
6117     my $changes;
6118     my $prefix = "VM $vmid";
6119
6120     # used and unused disks
6121     my $referenced = {};
6122
6123     # Note: it is allowed to define multiple storages with same path (alias), so
6124     # we need to check both 'volid' and real 'path' (two different volid can point
6125     # to the same path).
6126
6127     my $referencedpath = {};
6128
6129     # update size info
6130     PVE::QemuConfig->foreach_volume($conf, sub {
6131         my ($opt, $drive) = @_;
6132
6133         my $volid = $drive->{file};
6134         return if !$volid;
6135         my $volume = $volid_hash->{$volid};
6136
6137         # mark volid as "in-use" for next step
6138         $referenced->{$volid} = 1;
6139         if ($volume && (my $path = $volume->{path})) {
6140             $referencedpath->{$path} = 1;
6141         }
6142
6143         return if drive_is_cdrom($drive);
6144         return if !$volume;
6145
6146         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6147         if (defined($updated)) {
6148             $changes = 1;
6149             $conf->{$opt} = print_drive($updated);
6150             print "$prefix ($opt): $msg\n";
6151         }
6152     });
6153
6154     # remove 'unusedX' entry if volume is used
6155     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6156         my ($opt, $drive) = @_;
6157
6158         my $volid = $drive->{file};
6159         return if !$volid;
6160
6161         my $path;
6162         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6163         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6164             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6165             $changes = 1;
6166             delete $conf->{$opt};
6167         }
6168
6169         $referenced->{$volid} = 1;
6170         $referencedpath->{$path} = 1 if $path;
6171     });
6172
6173     foreach my $volid (sort keys %$volid_hash) {
6174         next if $volid =~ m/vm-$vmid-state-/;
6175         next if $referenced->{$volid};
6176         my $path = $volid_hash->{$volid}->{path};
6177         next if !$path; # just to be sure
6178         next if $referencedpath->{$path};
6179         $changes = 1;
6180         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6181         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6182         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6183     }
6184
6185     return $changes;
6186 }
6187
6188 sub rescan {
6189     my ($vmid, $nolock, $dryrun) = @_;
6190
6191     my $cfg = PVE::Storage::config();
6192
6193     print "rescan volumes...\n";
6194     my $volid_hash = scan_volids($cfg, $vmid);
6195
6196     my $updatefn =  sub {
6197         my ($vmid) = @_;
6198
6199         my $conf = PVE::QemuConfig->load_config($vmid);
6200
6201         PVE::QemuConfig->check_lock($conf);
6202
6203         my $vm_volids = {};
6204         foreach my $volid (keys %$volid_hash) {
6205             my $info = $volid_hash->{$volid};
6206             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6207         }
6208
6209         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6210
6211         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6212     };
6213
6214     if (defined($vmid)) {
6215         if ($nolock) {
6216             &$updatefn($vmid);
6217         } else {
6218             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6219         }
6220     } else {
6221         my $vmlist = config_list();
6222         foreach my $vmid (keys %$vmlist) {
6223             if ($nolock) {
6224                 &$updatefn($vmid);
6225             } else {
6226                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6227             }
6228         }
6229     }
6230 }
6231
6232 sub restore_proxmox_backup_archive {
6233     my ($archive, $vmid, $user, $options) = @_;
6234
6235     my $storecfg = PVE::Storage::config();
6236
6237     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6238     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6239
6240     my $fingerprint = $scfg->{fingerprint};
6241     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6242
6243     my $repo = PVE::PBSClient::get_repository($scfg);
6244
6245     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6246     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6247     local $ENV{PBS_PASSWORD} = $password;
6248     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6249
6250     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6251         PVE::Storage::parse_volname($storecfg, $archive);
6252
6253     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6254
6255     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6256
6257     my $tmpdir = "/var/tmp/vzdumptmp$$";
6258     rmtree $tmpdir;
6259     mkpath $tmpdir;
6260
6261     my $conffile = PVE::QemuConfig->config_file($vmid);
6262      # disable interrupts (always do cleanups)
6263     local $SIG{INT} =
6264         local $SIG{TERM} =
6265         local $SIG{QUIT} =
6266         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6267
6268     # Note: $oldconf is undef if VM does not exists
6269     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6270     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6271     my $new_conf_raw = '';
6272
6273     my $rpcenv = PVE::RPCEnvironment::get();
6274     my $devinfo = {};
6275
6276     eval {
6277         # enable interrupts
6278         local $SIG{INT} =
6279             local $SIG{TERM} =
6280             local $SIG{QUIT} =
6281             local $SIG{HUP} =
6282             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6283
6284         my $cfgfn = "$tmpdir/qemu-server.conf";
6285         my $firewall_config_fn = "$tmpdir/fw.conf";
6286         my $index_fn = "$tmpdir/index.json";
6287
6288         my $cmd = "restore";
6289
6290         my $param = [$pbs_backup_name, "index.json", $index_fn];
6291         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6292         my $index = PVE::Tools::file_get_contents($index_fn);
6293         $index = decode_json($index);
6294
6295         # print Dumper($index);
6296         foreach my $info (@{$index->{files}}) {
6297             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6298                 my $devname = $1;
6299                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6300                     $devinfo->{$devname}->{size} = $1;
6301                 } else {
6302                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6303                 }
6304             }
6305         }
6306
6307         my $is_qemu_server_backup = scalar(
6308             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6309         );
6310         if (!$is_qemu_server_backup) {
6311             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6312         }
6313         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6314
6315         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6316         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6317
6318         if ($has_firewall_config) {
6319             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6320             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6321
6322             my $pve_firewall_dir = '/etc/pve/firewall';
6323             mkdir $pve_firewall_dir; # make sure the dir exists
6324             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6325         }
6326
6327         my $fh = IO::File->new($cfgfn, "r") ||
6328             die "unable to read qemu-server.conf - $!\n";
6329
6330         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6331
6332         # fixme: rate limit?
6333
6334         # create empty/temp config
6335         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6336
6337         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6338
6339         # allocate volumes
6340         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6341
6342         if (!$options->{live}) {
6343             foreach my $virtdev (sort keys %$virtdev_hash) {
6344                 my $d = $virtdev_hash->{$virtdev};
6345                 next if $d->{is_cloudinit}; # no need to restore cloudinit
6346
6347                 my $volid = $d->{volid};
6348
6349                 my $path = PVE::Storage::path($storecfg, $volid);
6350
6351                 my $pbs_restore_cmd = [
6352                     '/usr/bin/pbs-restore',
6353                     '--repository', $repo,
6354                     $pbs_backup_name,
6355                     "$d->{devname}.img.fidx",
6356                     $path,
6357                     '--verbose',
6358                     ];
6359
6360                 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6361                 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6362
6363                 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6364                     push @$pbs_restore_cmd, '--skip-zero';
6365                 }
6366
6367                 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6368                 print "restore proxmox backup image: $dbg_cmdstring\n";
6369                 run_command($pbs_restore_cmd);
6370             }
6371         }
6372
6373         $fh->seek(0, 0) || die "seek failed - $!\n";
6374
6375         my $cookie = { netcount => 0 };
6376         while (defined(my $line = <$fh>)) {
6377             $new_conf_raw .= restore_update_config_line(
6378                 $cookie,
6379                 $map,
6380                 $line,
6381                 $options->{unique},
6382             );
6383         }
6384
6385         $fh->close();
6386     };
6387     my $err = $@;
6388
6389     if ($err || !$options->{live}) {
6390         $restore_deactivate_volumes->($storecfg, $devinfo);
6391     }
6392
6393     rmtree $tmpdir;
6394
6395     if ($err) {
6396         $restore_destroy_volumes->($storecfg, $devinfo);
6397         die $err;
6398     }
6399
6400     if ($options->{live}) {
6401         # keep lock during live-restore
6402         $new_conf_raw .= "\nlock: create";
6403     }
6404
6405     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6406
6407     PVE::Cluster::cfs_update(); # make sure we read new file
6408
6409     eval { rescan($vmid, 1); };
6410     warn $@ if $@;
6411
6412     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6413
6414     if ($options->{live}) {
6415         # enable interrupts
6416         local $SIG{INT} =
6417             local $SIG{TERM} =
6418             local $SIG{QUIT} =
6419             local $SIG{HUP} =
6420             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6421
6422         my $conf = PVE::QemuConfig->load_config($vmid);
6423         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6424
6425         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6426
6427         PVE::QemuConfig->remove_lock($vmid, "create");
6428     }
6429 }
6430
6431 sub pbs_live_restore {
6432     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6433
6434     print "starting VM for live-restore\n";
6435     print "repository: '$repo', snapshot: '$snap'\n";
6436
6437     my $pbs_backing = {};
6438     for my $ds (keys %$restored_disks) {
6439         $ds =~ m/^drive-(.*)$/;
6440         my $confname = $1;
6441         $pbs_backing->{$confname} = {
6442             repository => $repo,
6443             snapshot => $snap,
6444             archive => "$ds.img.fidx",
6445         };
6446         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6447
6448         my $drive = parse_drive($confname, $conf->{$confname});
6449         print "restoring '$ds' to '$drive->{file}'\n";
6450     }
6451
6452     my $drives_streamed = 0;
6453     eval {
6454         # make sure HA doesn't interrupt our restore by stopping the VM
6455         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6456             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6457         }
6458
6459         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6460         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6461         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6462
6463         my $qmeventd_fd = register_qmeventd_handle($vmid);
6464
6465         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6466         # this will effectively collapse the backing image chain consisting of
6467         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6468         # removes itself once all backing images vanish with 'auto-remove=on')
6469         my $jobs = {};
6470         for my $ds (sort keys %$restored_disks) {
6471             my $job_id = "restore-$ds";
6472             mon_cmd($vmid, 'block-stream',
6473                 'job-id' => $job_id,
6474                 device => "$ds",
6475             );
6476             $jobs->{$job_id} = {};
6477         }
6478
6479         mon_cmd($vmid, 'cont');
6480         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6481
6482         print "restore-drive jobs finished successfully, removing all tracking block devices"
6483             ." to disconnect from Proxmox Backup Server\n";
6484
6485         for my $ds (sort keys %$restored_disks) {
6486             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6487         }
6488
6489         close($qmeventd_fd);
6490     };
6491
6492     my $err = $@;
6493
6494     if ($err) {
6495         warn "An error occured during live-restore: $err\n";
6496         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6497         die "live-restore failed\n";
6498     }
6499 }
6500
6501 sub restore_vma_archive {
6502     my ($archive, $vmid, $user, $opts, $comp) = @_;
6503
6504     my $readfrom = $archive;
6505
6506     my $cfg = PVE::Storage::config();
6507     my $commands = [];
6508     my $bwlimit = $opts->{bwlimit};
6509
6510     my $dbg_cmdstring = '';
6511     my $add_pipe = sub {
6512         my ($cmd) = @_;
6513         push @$commands, $cmd;
6514         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6515         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6516         $readfrom = '-';
6517     };
6518
6519     my $input = undef;
6520     if ($archive eq '-') {
6521         $input = '<&STDIN';
6522     } else {
6523         # If we use a backup from a PVE defined storage we also consider that
6524         # storage's rate limit:
6525         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6526         if (defined($volid)) {
6527             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6528             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6529             if ($readlimit) {
6530                 print STDERR "applying read rate limit: $readlimit\n";
6531                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6532                 $add_pipe->($cstream);
6533             }
6534         }
6535     }
6536
6537     if ($comp) {
6538         my $info = PVE::Storage::decompressor_info('vma', $comp);
6539         my $cmd = $info->{decompressor};
6540         push @$cmd, $readfrom;
6541         $add_pipe->($cmd);
6542     }
6543
6544     my $tmpdir = "/var/tmp/vzdumptmp$$";
6545     rmtree $tmpdir;
6546
6547     # disable interrupts (always do cleanups)
6548     local $SIG{INT} =
6549         local $SIG{TERM} =
6550         local $SIG{QUIT} =
6551         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6552
6553     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6554     POSIX::mkfifo($mapfifo, 0600);
6555     my $fifofh;
6556     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6557
6558     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6559
6560     my $oldtimeout;
6561     my $timeout = 5;
6562
6563     my $devinfo = {};
6564
6565     my $rpcenv = PVE::RPCEnvironment::get();
6566
6567     my $conffile = PVE::QemuConfig->config_file($vmid);
6568
6569     # Note: $oldconf is undef if VM does not exist
6570     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6571     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6572     my $new_conf_raw = '';
6573
6574     my %storage_limits;
6575
6576     my $print_devmap = sub {
6577         my $cfgfn = "$tmpdir/qemu-server.conf";
6578
6579         # we can read the config - that is already extracted
6580         my $fh = IO::File->new($cfgfn, "r") ||
6581             die "unable to read qemu-server.conf - $!\n";
6582
6583         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6584         if (-f $fwcfgfn) {
6585             my $pve_firewall_dir = '/etc/pve/firewall';
6586             mkdir $pve_firewall_dir; # make sure the dir exists
6587             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6588         }
6589
6590         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6591
6592         foreach my $info (values %{$virtdev_hash}) {
6593             my $storeid = $info->{storeid};
6594             next if defined($storage_limits{$storeid});
6595
6596             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6597             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6598             $storage_limits{$storeid} = $limit * 1024;
6599         }
6600
6601         foreach my $devname (keys %$devinfo) {
6602             die "found no device mapping information for device '$devname'\n"
6603                 if !$devinfo->{$devname}->{virtdev};
6604         }
6605
6606         # create empty/temp config
6607         if ($oldconf) {
6608             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6609             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6610         }
6611
6612         # allocate volumes
6613         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6614
6615         # print restore information to $fifofh
6616         foreach my $virtdev (sort keys %$virtdev_hash) {
6617             my $d = $virtdev_hash->{$virtdev};
6618             next if $d->{is_cloudinit}; # no need to restore cloudinit
6619
6620             my $storeid = $d->{storeid};
6621             my $volid = $d->{volid};
6622
6623             my $map_opts = '';
6624             if (my $limit = $storage_limits{$storeid}) {
6625                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6626             }
6627
6628             my $write_zeros = 1;
6629             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6630                 $write_zeros = 0;
6631             }
6632
6633             my $path = PVE::Storage::path($cfg, $volid);
6634
6635             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6636
6637             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6638         }
6639
6640         $fh->seek(0, 0) || die "seek failed - $!\n";
6641
6642         my $cookie = { netcount => 0 };
6643         while (defined(my $line = <$fh>)) {
6644             $new_conf_raw .= restore_update_config_line(
6645                 $cookie,
6646                 $map,
6647                 $line,
6648                 $opts->{unique},
6649             );
6650         }
6651
6652         $fh->close();
6653     };
6654
6655     eval {
6656         # enable interrupts
6657         local $SIG{INT} =
6658             local $SIG{TERM} =
6659             local $SIG{QUIT} =
6660             local $SIG{HUP} =
6661             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6662         local $SIG{ALRM} = sub { die "got timeout\n"; };
6663
6664         $oldtimeout = alarm($timeout);
6665
6666         my $parser = sub {
6667             my $line = shift;
6668
6669             print "$line\n";
6670
6671             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6672                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6673                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6674             } elsif ($line =~ m/^CTIME: /) {
6675                 # we correctly received the vma config, so we can disable
6676                 # the timeout now for disk allocation (set to 10 minutes, so
6677                 # that we always timeout if something goes wrong)
6678                 alarm(600);
6679                 &$print_devmap();
6680                 print $fifofh "done\n";
6681                 my $tmp = $oldtimeout || 0;
6682                 $oldtimeout = undef;
6683                 alarm($tmp);
6684                 close($fifofh);
6685                 $fifofh = undef;
6686             }
6687         };
6688
6689         print "restore vma archive: $dbg_cmdstring\n";
6690         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6691     };
6692     my $err = $@;
6693
6694     alarm($oldtimeout) if $oldtimeout;
6695
6696     $restore_deactivate_volumes->($cfg, $devinfo);
6697
6698     close($fifofh) if $fifofh;
6699     unlink $mapfifo;
6700     rmtree $tmpdir;
6701
6702     if ($err) {
6703         $restore_destroy_volumes->($cfg, $devinfo);
6704         die $err;
6705     }
6706
6707     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6708
6709     PVE::Cluster::cfs_update(); # make sure we read new file
6710
6711     eval { rescan($vmid, 1); };
6712     warn $@ if $@;
6713
6714     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6715 }
6716
6717 sub restore_tar_archive {
6718     my ($archive, $vmid, $user, $opts) = @_;
6719
6720     if ($archive ne '-') {
6721         my $firstfile = tar_archive_read_firstfile($archive);
6722         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6723             if $firstfile ne 'qemu-server.conf';
6724     }
6725
6726     my $storecfg = PVE::Storage::config();
6727
6728     # avoid zombie disks when restoring over an existing VM -> cleanup first
6729     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6730     # skiplock=1 because qmrestore has set the 'create' lock itself already
6731     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6732     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6733
6734     my $tocmd = "/usr/lib/qemu-server/qmextract";
6735
6736     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6737     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6738     $tocmd .= ' --prealloc' if $opts->{prealloc};
6739     $tocmd .= ' --info' if $opts->{info};
6740
6741     # tar option "xf" does not autodetect compression when read from STDIN,
6742     # so we pipe to zcat
6743     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6744         PVE::Tools::shellquote("--to-command=$tocmd");
6745
6746     my $tmpdir = "/var/tmp/vzdumptmp$$";
6747     mkpath $tmpdir;
6748
6749     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6750     local $ENV{VZDUMP_VMID} = $vmid;
6751     local $ENV{VZDUMP_USER} = $user;
6752
6753     my $conffile = PVE::QemuConfig->config_file($vmid);
6754     my $new_conf_raw = '';
6755
6756     # disable interrupts (always do cleanups)
6757     local $SIG{INT} =
6758         local $SIG{TERM} =
6759         local $SIG{QUIT} =
6760         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6761
6762     eval {
6763         # enable interrupts
6764         local $SIG{INT} =
6765             local $SIG{TERM} =
6766             local $SIG{QUIT} =
6767             local $SIG{HUP} =
6768             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6769
6770         if ($archive eq '-') {
6771             print "extracting archive from STDIN\n";
6772             run_command($cmd, input => "<&STDIN");
6773         } else {
6774             print "extracting archive '$archive'\n";
6775             run_command($cmd);
6776         }
6777
6778         return if $opts->{info};
6779
6780         # read new mapping
6781         my $map = {};
6782         my $statfile = "$tmpdir/qmrestore.stat";
6783         if (my $fd = IO::File->new($statfile, "r")) {
6784             while (defined (my $line = <$fd>)) {
6785                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6786                     $map->{$1} = $2 if $1;
6787                 } else {
6788                     print STDERR "unable to parse line in statfile - $line\n";
6789                 }
6790             }
6791             $fd->close();
6792         }
6793
6794         my $confsrc = "$tmpdir/qemu-server.conf";
6795
6796         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6797
6798         my $cookie = { netcount => 0 };
6799         while (defined (my $line = <$srcfd>)) {
6800             $new_conf_raw .= restore_update_config_line(
6801                 $cookie,
6802                 $map,
6803                 $line,
6804                 $opts->{unique},
6805             );
6806         }
6807
6808         $srcfd->close();
6809     };
6810     if (my $err = $@) {
6811         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6812         die $err;
6813     }
6814
6815     rmtree $tmpdir;
6816
6817     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6818
6819     PVE::Cluster::cfs_update(); # make sure we read new file
6820
6821     eval { rescan($vmid, 1); };
6822     warn $@ if $@;
6823 };
6824
6825 sub foreach_storage_used_by_vm {
6826     my ($conf, $func) = @_;
6827
6828     my $sidhash = {};
6829
6830     PVE::QemuConfig->foreach_volume($conf, sub {
6831         my ($ds, $drive) = @_;
6832         return if drive_is_cdrom($drive);
6833
6834         my $volid = $drive->{file};
6835
6836         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6837         $sidhash->{$sid} = $sid if $sid;
6838     });
6839
6840     foreach my $sid (sort keys %$sidhash) {
6841         &$func($sid);
6842     }
6843 }
6844
6845 my $qemu_snap_storage = {
6846     rbd => 1,
6847 };
6848 sub do_snapshots_with_qemu {
6849     my ($storecfg, $volid) = @_;
6850
6851     my $storage_name = PVE::Storage::parse_volume_id($volid);
6852     my $scfg = $storecfg->{ids}->{$storage_name};
6853     die "could not find storage '$storage_name'\n" if !defined($scfg);
6854
6855     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6856         return 1;
6857     }
6858
6859     if ($volid =~ m/\.(qcow2|qed)$/){
6860         return 1;
6861     }
6862
6863     return;
6864 }
6865
6866 sub qga_check_running {
6867     my ($vmid, $nowarn) = @_;
6868
6869     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6870     if ($@) {
6871         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6872         return 0;
6873     }
6874     return 1;
6875 }
6876
6877 sub template_create {
6878     my ($vmid, $conf, $disk) = @_;
6879
6880     my $storecfg = PVE::Storage::config();
6881
6882     PVE::QemuConfig->foreach_volume($conf, sub {
6883         my ($ds, $drive) = @_;
6884
6885         return if drive_is_cdrom($drive);
6886         return if $disk && $ds ne $disk;
6887
6888         my $volid = $drive->{file};
6889         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6890
6891         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6892         $drive->{file} = $voliddst;
6893         $conf->{$ds} = print_drive($drive);
6894         PVE::QemuConfig->write_config($vmid, $conf);
6895     });
6896 }
6897
6898 sub convert_iscsi_path {
6899     my ($path) = @_;
6900
6901     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6902         my $portal = $1;
6903         my $target = $2;
6904         my $lun = $3;
6905
6906         my $initiator_name = get_initiator_name();
6907
6908         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6909                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6910     }
6911
6912     die "cannot convert iscsi path '$path', unkown format\n";
6913 }
6914
6915 sub qemu_img_convert {
6916     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6917
6918     my $storecfg = PVE::Storage::config();
6919     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6920     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6921
6922     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6923
6924     my $cachemode;
6925     my $src_path;
6926     my $src_is_iscsi = 0;
6927     my $src_format;
6928
6929     if ($src_storeid) {
6930         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6931         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6932         $src_format = qemu_img_format($src_scfg, $src_volname);
6933         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6934         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6935         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6936     } elsif (-f $src_volid) {
6937         $src_path = $src_volid;
6938         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6939             $src_format = $1;
6940         }
6941     }
6942
6943     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6944
6945     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6946     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6947     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6948     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6949
6950     my $cmd = [];
6951     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6952     push @$cmd, '-l', "snapshot.name=$snapname"
6953         if $snapname && $src_format && $src_format eq "qcow2";
6954     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6955     push @$cmd, '-T', $cachemode if defined($cachemode);
6956
6957     if ($src_is_iscsi) {
6958         push @$cmd, '--image-opts';
6959         $src_path = convert_iscsi_path($src_path);
6960     } elsif ($src_format) {
6961         push @$cmd, '-f', $src_format;
6962     }
6963
6964     if ($dst_is_iscsi) {
6965         push @$cmd, '--target-image-opts';
6966         $dst_path = convert_iscsi_path($dst_path);
6967     } else {
6968         push @$cmd, '-O', $dst_format;
6969     }
6970
6971     push @$cmd, $src_path;
6972
6973     if (!$dst_is_iscsi && $is_zero_initialized) {
6974         push @$cmd, "zeroinit:$dst_path";
6975     } else {
6976         push @$cmd, $dst_path;
6977     }
6978
6979     my $parser = sub {
6980         my $line = shift;
6981         if($line =~ m/\((\S+)\/100\%\)/){
6982             my $percent = $1;
6983             my $transferred = int($size * $percent / 100);
6984             my $total_h = render_bytes($size, 1);
6985             my $transferred_h = render_bytes($transferred, 1);
6986
6987             print "transferred $transferred_h of $total_h ($percent%)\n";
6988         }
6989
6990     };
6991
6992     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6993     my $err = $@;
6994     die "copy failed: $err" if $err;
6995 }
6996
6997 sub qemu_img_format {
6998     my ($scfg, $volname) = @_;
6999
7000     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7001         return $1;
7002     } else {
7003         return "raw";
7004     }
7005 }
7006
7007 sub qemu_drive_mirror {
7008     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7009
7010     $jobs = {} if !$jobs;
7011
7012     my $qemu_target;
7013     my $format;
7014     $jobs->{"drive-$drive"} = {};
7015
7016     if ($dst_volid =~ /^nbd:/) {
7017         $qemu_target = $dst_volid;
7018         $format = "nbd";
7019     } else {
7020         my $storecfg = PVE::Storage::config();
7021         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7022
7023         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7024
7025         $format = qemu_img_format($dst_scfg, $dst_volname);
7026
7027         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7028
7029         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7030     }
7031
7032     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7033     $opts->{format} = $format if $format;
7034
7035     if (defined($src_bitmap)) {
7036         $opts->{sync} = 'incremental';
7037         $opts->{bitmap} = $src_bitmap;
7038         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7039     }
7040
7041     if (defined($bwlimit)) {
7042         $opts->{speed} = $bwlimit * 1024;
7043         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7044     } else {
7045         print "drive mirror is starting for drive-$drive\n";
7046     }
7047
7048     # if a job already runs for this device we get an error, catch it for cleanup
7049     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7050     if (my $err = $@) {
7051         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7052         warn "$@\n" if $@;
7053         die "mirroring error: $err\n";
7054     }
7055
7056     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7057 }
7058
7059 # $completion can be either
7060 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7061 # 'cancel': wait until all jobs are ready, block-job-cancel them
7062 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7063 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7064 sub qemu_drive_mirror_monitor {
7065     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7066
7067     $completion //= 'complete';
7068     $op //= "mirror";
7069
7070     eval {
7071         my $err_complete = 0;
7072
7073         my $starttime = time ();
7074         while (1) {
7075             die "block job ('$op') timed out\n" if $err_complete > 300;
7076
7077             my $stats = mon_cmd($vmid, "query-block-jobs");
7078             my $ctime = time();
7079
7080             my $running_jobs = {};
7081             for my $stat (@$stats) {
7082                 next if $stat->{type} ne $op;
7083                 $running_jobs->{$stat->{device}} = $stat;
7084             }
7085
7086             my $readycounter = 0;
7087
7088             for my $job_id (sort keys %$jobs) {
7089                 my $job = $running_jobs->{$job_id};
7090
7091                 my $vanished = !defined($job);
7092                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7093                 if($complete || ($vanished && $completion eq 'auto')) {
7094                     print "$job_id: $op-job finished\n";
7095                     delete $jobs->{$job_id};
7096                     next;
7097                 }
7098
7099                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7100
7101                 my $busy = $job->{busy};
7102                 my $ready = $job->{ready};
7103                 if (my $total = $job->{len}) {
7104                     my $transferred = $job->{offset} || 0;
7105                     my $remaining = $total - $transferred;
7106                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7107
7108                     my $duration = $ctime - $starttime;
7109                     my $total_h = render_bytes($total, 1);
7110                     my $transferred_h = render_bytes($transferred, 1);
7111
7112                     my $status = sprintf(
7113                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7114                         render_duration($duration),
7115                     );
7116
7117                     if ($ready) {
7118                         if ($busy) {
7119                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7120                         } else {
7121                             $status .= ", ready";
7122                         }
7123                     }
7124                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7125                     $jobs->{$job_id}->{ready} = $ready;
7126                 }
7127
7128                 $readycounter++ if $job->{ready};
7129             }
7130
7131             last if scalar(keys %$jobs) == 0;
7132
7133             if ($readycounter == scalar(keys %$jobs)) {
7134                 print "all '$op' jobs are ready\n";
7135
7136                 # do the complete later (or has already been done)
7137                 last if $completion eq 'skip' || $completion eq 'auto';
7138
7139                 if ($vmiddst && $vmiddst != $vmid) {
7140                     my $agent_running = $qga && qga_check_running($vmid);
7141                     if ($agent_running) {
7142                         print "freeze filesystem\n";
7143                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7144                     } else {
7145                         print "suspend vm\n";
7146                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7147                     }
7148
7149                     # if we clone a disk for a new target vm, we don't switch the disk
7150                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7151
7152                     if ($agent_running) {
7153                         print "unfreeze filesystem\n";
7154                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7155                     } else {
7156                         print "resume vm\n";
7157                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7158                     }
7159
7160                     last;
7161                 } else {
7162
7163                     for my $job_id (sort keys %$jobs) {
7164                         # try to switch the disk if source and destination are on the same guest
7165                         print "$job_id: Completing block job_id...\n";
7166
7167                         my $op;
7168                         if ($completion eq 'complete') {
7169                             $op = 'block-job-complete';
7170                         } elsif ($completion eq 'cancel') {
7171                             $op = 'block-job-cancel';
7172                         } else {
7173                             die "invalid completion value: $completion\n";
7174                         }
7175                         eval { mon_cmd($vmid, $op, device => $job_id) };
7176                         if ($@ =~ m/cannot be completed/) {
7177                             print "$job_id: block job cannot be completed, trying again.\n";
7178                             $err_complete++;
7179                         }else {
7180                             print "$job_id: Completed successfully.\n";
7181                             $jobs->{$job_id}->{complete} = 1;
7182                         }
7183                     }
7184                 }
7185             }
7186             sleep 1;
7187         }
7188     };
7189     my $err = $@;
7190
7191     if ($err) {
7192         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7193         die "block job ($op) error: $err";
7194     }
7195 }
7196
7197 sub qemu_blockjobs_cancel {
7198     my ($vmid, $jobs) = @_;
7199
7200     foreach my $job (keys %$jobs) {
7201         print "$job: Cancelling block job\n";
7202         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7203         $jobs->{$job}->{cancel} = 1;
7204     }
7205
7206     while (1) {
7207         my $stats = mon_cmd($vmid, "query-block-jobs");
7208
7209         my $running_jobs = {};
7210         foreach my $stat (@$stats) {
7211             $running_jobs->{$stat->{device}} = $stat;
7212         }
7213
7214         foreach my $job (keys %$jobs) {
7215
7216             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7217                 print "$job: Done.\n";
7218                 delete $jobs->{$job};
7219             }
7220         }
7221
7222         last if scalar(keys %$jobs) == 0;
7223
7224         sleep 1;
7225     }
7226 }
7227
7228 sub clone_disk {
7229     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7230         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7231
7232     my $newvolid;
7233
7234     if (!$full) {
7235         print "create linked clone of drive $drivename ($drive->{file})\n";
7236         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7237         push @$newvollist, $newvolid;
7238     } else {
7239
7240         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7241         $storeid = $storage if $storage;
7242
7243         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7244
7245         print "create full clone of drive $drivename ($drive->{file})\n";
7246         my $name = undef;
7247         my $size = undef;
7248         if (drive_is_cloudinit($drive)) {
7249             $name = "vm-$newvmid-cloudinit";
7250             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7251             if ($scfg->{path}) {
7252                 $name .= ".$dst_format";
7253             }
7254             $snapname = undef;
7255             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7256         } elsif ($drivename eq 'efidisk0') {
7257             $size = get_efivars_size($conf);
7258         } else {
7259             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7260         }
7261         $newvolid = PVE::Storage::vdisk_alloc(
7262             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7263         );
7264         push @$newvollist, $newvolid;
7265
7266         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7267
7268         if (drive_is_cloudinit($drive)) {
7269             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7270             # if this is the case, we have to complete any block-jobs still there from
7271             # previous drive-mirrors
7272             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7273                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7274             }
7275             goto no_data_clone;
7276         }
7277
7278         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7279         if (!$running || $snapname) {
7280             # TODO: handle bwlimits
7281             if ($drivename eq 'efidisk0') {
7282                 # the relevant data on the efidisk may be smaller than the source
7283                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7284                 # that is given by the OVMF_VARS.fd
7285                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7286                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7287
7288                 # better for Ceph if block size is not too small, see bug #3324
7289                 my $bs = 1024*1024;
7290
7291                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7292                     "if=$src_path", "of=$dst_path"]);
7293             } else {
7294                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7295             }
7296         } else {
7297
7298             my $kvmver = get_running_qemu_version ($vmid);
7299             if (!min_version($kvmver, 2, 7)) {
7300                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7301                     if $drive->{iothread};
7302             }
7303
7304             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7305                 $completion, $qga, $bwlimit);
7306         }
7307     }
7308
7309 no_data_clone:
7310     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7311
7312     my $disk = $drive;
7313     $disk->{format} = undef;
7314     $disk->{file} = $newvolid;
7315     $disk->{size} = $size if defined($size);
7316
7317     return $disk;
7318 }
7319
7320 sub get_running_qemu_version {
7321     my ($vmid) = @_;
7322     my $res = mon_cmd($vmid, "query-version");
7323     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7324 }
7325
7326 sub qemu_use_old_bios_files {
7327     my ($machine_type) = @_;
7328
7329     return if !$machine_type;
7330
7331     my $use_old_bios_files = undef;
7332
7333     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7334         $machine_type = $1;
7335         $use_old_bios_files = 1;
7336     } else {
7337         my $version = extract_version($machine_type, kvm_user_version());
7338         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7339         # load new efi bios files on migration. So this hack is required to allow
7340         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7341         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7342         $use_old_bios_files = !min_version($version, 2, 4);
7343     }
7344
7345     return ($use_old_bios_files, $machine_type);
7346 }
7347
7348 sub get_efivars_size {
7349     my ($conf) = @_;
7350     my $arch = get_vm_arch($conf);
7351     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7352     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7353     return -s $ovmf_vars;
7354 }
7355
7356 sub update_efidisk_size {
7357     my ($conf) = @_;
7358
7359     return if !defined($conf->{efidisk0});
7360
7361     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7362     $disk->{size} = get_efivars_size($conf);
7363     $conf->{efidisk0} = print_drive($disk);
7364
7365     return;
7366 }
7367
7368 sub create_efidisk($$$$$) {
7369     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7370
7371     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7372     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7373
7374     my $vars_size_b = -s $ovmf_vars;
7375     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7376     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7377     PVE::Storage::activate_volumes($storecfg, [$volid]);
7378
7379     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7380     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7381
7382     return ($volid, $size/1024);
7383 }
7384
7385 sub vm_iothreads_list {
7386     my ($vmid) = @_;
7387
7388     my $res = mon_cmd($vmid, 'query-iothreads');
7389
7390     my $iothreads = {};
7391     foreach my $iothread (@$res) {
7392         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7393     }
7394
7395     return $iothreads;
7396 }
7397
7398 sub scsihw_infos {
7399     my ($conf, $drive) = @_;
7400
7401     my $maxdev = 0;
7402
7403     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7404         $maxdev = 7;
7405     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7406         $maxdev = 1;
7407     } else {
7408         $maxdev = 256;
7409     }
7410
7411     my $controller = int($drive->{index} / $maxdev);
7412     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7413         ? "virtioscsi"
7414         : "scsihw";
7415
7416     return ($maxdev, $controller, $controller_prefix);
7417 }
7418
7419 sub windows_version {
7420     my ($ostype) = @_;
7421
7422     return 0 if !$ostype;
7423
7424     my $winversion = 0;
7425
7426     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7427         $winversion = 5;
7428     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7429         $winversion = 6;
7430     } elsif ($ostype =~ m/^win(\d+)$/) {
7431         $winversion = $1;
7432     }
7433
7434     return $winversion;
7435 }
7436
7437 sub resolve_dst_disk_format {
7438         my ($storecfg, $storeid, $src_volname, $format) = @_;
7439         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7440
7441         if (!$format) {
7442             # if no target format is specified, use the source disk format as hint
7443             if ($src_volname) {
7444                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7445                 $format = qemu_img_format($scfg, $src_volname);
7446             } else {
7447                 return $defFormat;
7448             }
7449         }
7450
7451         # test if requested format is supported - else use default
7452         my $supported = grep { $_ eq $format } @$validFormats;
7453         $format = $defFormat if !$supported;
7454         return $format;
7455 }
7456
7457 # NOTE: if this logic changes, please update docs & possibly gui logic
7458 sub find_vmstate_storage {
7459     my ($conf, $storecfg) = @_;
7460
7461     # first, return storage from conf if set
7462     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7463
7464     my ($target, $shared, $local);
7465
7466     foreach_storage_used_by_vm($conf, sub {
7467         my ($sid) = @_;
7468         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7469         my $dst = $scfg->{shared} ? \$shared : \$local;
7470         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7471     });
7472
7473     # second, use shared storage where VM has at least one disk
7474     # third, use local storage where VM has at least one disk
7475     # fall back to local storage
7476     $target = $shared // $local // 'local';
7477
7478     return $target;
7479 }
7480
7481 sub generate_uuid {
7482     my ($uuid, $uuid_str);
7483     UUID::generate($uuid);
7484     UUID::unparse($uuid, $uuid_str);
7485     return $uuid_str;
7486 }
7487
7488 sub generate_smbios1_uuid {
7489     return "uuid=".generate_uuid();
7490 }
7491
7492 sub nbd_stop {
7493     my ($vmid) = @_;
7494
7495     mon_cmd($vmid, 'nbd-server-stop');
7496 }
7497
7498 sub create_reboot_request {
7499     my ($vmid) = @_;
7500     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7501         or die "failed to create reboot trigger file: $!\n";
7502     close($fh);
7503 }
7504
7505 sub clear_reboot_request {
7506     my ($vmid) = @_;
7507     my $path = "/run/qemu-server/$vmid.reboot";
7508     my $res = 0;
7509
7510     $res = unlink($path);
7511     die "could not remove reboot request for $vmid: $!"
7512         if !$res && $! != POSIX::ENOENT;
7513
7514     return $res;
7515 }
7516
7517 sub bootorder_from_legacy {
7518     my ($conf, $bootcfg) = @_;
7519
7520     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7521     my $bootindex_hash = {};
7522     my $i = 1;
7523     foreach my $o (split(//, $boot)) {
7524         $bootindex_hash->{$o} = $i*100;
7525         $i++;
7526     }
7527
7528     my $bootorder = {};
7529
7530     PVE::QemuConfig->foreach_volume($conf, sub {
7531         my ($ds, $drive) = @_;
7532
7533         if (drive_is_cdrom ($drive, 1)) {
7534             if ($bootindex_hash->{d}) {
7535                 $bootorder->{$ds} = $bootindex_hash->{d};
7536                 $bootindex_hash->{d} += 1;
7537             }
7538         } elsif ($bootindex_hash->{c}) {
7539             $bootorder->{$ds} = $bootindex_hash->{c}
7540                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7541             $bootindex_hash->{c} += 1;
7542         }
7543     });
7544
7545     if ($bootindex_hash->{n}) {
7546         for (my $i = 0; $i < $MAX_NETS; $i++) {
7547             my $netname = "net$i";
7548             next if !$conf->{$netname};
7549             $bootorder->{$netname} = $bootindex_hash->{n};
7550             $bootindex_hash->{n} += 1;
7551         }
7552     }
7553
7554     return $bootorder;
7555 }
7556
7557 # Generate default device list for 'boot: order=' property. Matches legacy
7558 # default boot order, but with explicit device names. This is important, since
7559 # the fallback for when neither 'order' nor the old format is specified relies
7560 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7561 sub get_default_bootdevices {
7562     my ($conf) = @_;
7563
7564     my @ret = ();
7565
7566     # harddisk
7567     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7568     push @ret, $first if $first;
7569
7570     # cdrom
7571     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7572     push @ret, $first if $first;
7573
7574     # network
7575     for (my $i = 0; $i < $MAX_NETS; $i++) {
7576         my $netname = "net$i";
7577         next if !$conf->{$netname};
7578         push @ret, $netname;
7579         last;
7580     }
7581
7582     return \@ret;
7583 }
7584
7585 sub device_bootorder {
7586     my ($conf) = @_;
7587
7588     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7589
7590     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7591
7592     my $bootorder = {};
7593     if (!defined($boot) || $boot->{legacy}) {
7594         $bootorder = bootorder_from_legacy($conf, $boot);
7595     } elsif ($boot->{order}) {
7596         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7597         for my $dev (PVE::Tools::split_list($boot->{order})) {
7598             $bootorder->{$dev} = $i++;
7599         }
7600     }
7601
7602     return $bootorder;
7603 }
7604
7605 sub register_qmeventd_handle {
7606     my ($vmid) = @_;
7607
7608     my $fh;
7609     my $peer = "/var/run/qmeventd.sock";
7610     my $count = 0;
7611
7612     for (;;) {
7613         $count++;
7614         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7615         last if $fh;
7616         if ($! != EINTR && $! != EAGAIN) {
7617             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7618         }
7619         if ($count > 4) {
7620             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7621               . "after $count retries\n";
7622         }
7623         usleep(25000);
7624     }
7625
7626     # send handshake to mark VM as backing up
7627     print $fh to_json({vzdump => {vmid => "$vmid"}});
7628
7629     # return handle to be closed later when inhibit is no longer required
7630     return $fh;
7631 }
7632
7633 # bash completion helper
7634
7635 sub complete_backup_archives {
7636     my ($cmdname, $pname, $cvalue) = @_;
7637
7638     my $cfg = PVE::Storage::config();
7639
7640     my $storeid;
7641
7642     if ($cvalue =~ m/^([^:]+):/) {
7643         $storeid = $1;
7644     }
7645
7646     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7647
7648     my $res = [];
7649     foreach my $id (keys %$data) {
7650         foreach my $item (@{$data->{$id}}) {
7651             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7652             push @$res, $item->{volid} if defined($item->{volid});
7653         }
7654     }
7655
7656     return $res;
7657 }
7658
7659 my $complete_vmid_full = sub {
7660     my ($running) = @_;
7661
7662     my $idlist = vmstatus();
7663
7664     my $res = [];
7665
7666     foreach my $id (keys %$idlist) {
7667         my $d = $idlist->{$id};
7668         if (defined($running)) {
7669             next if $d->{template};
7670             next if $running && $d->{status} ne 'running';
7671             next if !$running && $d->{status} eq 'running';
7672         }
7673         push @$res, $id;
7674
7675     }
7676     return $res;
7677 };
7678
7679 sub complete_vmid {
7680     return &$complete_vmid_full();
7681 }
7682
7683 sub complete_vmid_stopped {
7684     return &$complete_vmid_full(0);
7685 }
7686
7687 sub complete_vmid_running {
7688     return &$complete_vmid_full(1);
7689 }
7690
7691 sub complete_storage {
7692
7693     my $cfg = PVE::Storage::config();
7694     my $ids = $cfg->{ids};
7695
7696     my $res = [];
7697     foreach my $sid (keys %$ids) {
7698         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7699         next if !$ids->{$sid}->{content}->{images};
7700         push @$res, $sid;
7701     }
7702
7703     return $res;
7704 }
7705
7706 sub complete_migration_storage {
7707     my ($cmd, $param, $current_value, $all_args) = @_;
7708
7709     my $targetnode = @$all_args[1];
7710
7711     my $cfg = PVE::Storage::config();
7712     my $ids = $cfg->{ids};
7713
7714     my $res = [];
7715     foreach my $sid (keys %$ids) {
7716         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7717         next if !$ids->{$sid}->{content}->{images};
7718         push @$res, $sid;
7719     }
7720
7721     return $res;
7722 }
7723
7724 sub vm_is_paused {
7725     my ($vmid) = @_;
7726     my $qmpstatus = eval {
7727         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7728         mon_cmd($vmid, "query-status");
7729     };
7730     warn "$@\n" if $@;
7731     return $qmpstatus && $qmpstatus->{status} eq "paused";
7732 }
7733
7734 1;