PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::RPCEnvironment;
  38 use PVE::Storage;
  39 use PVE::SysFSTools;
  40 use PVE::Systemd;
  41 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  42
  43 use PVE::QMPClient;
  44 use PVE::QemuConfig;
  45 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  46 use PVE::QemuServer::Cloudinit;
  47 use PVE::QemuServer::CGroup;
  48 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  49 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  50 use PVE::QemuServer::Machine;
  51 use PVE::QemuServer::Memory;
  52 use PVE::QemuServer::Monitor qw(mon_cmd);
  53 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  54 use PVE::QemuServer::USB qw(parse_usb_device);
  55
  56 my $have_sdn;
  57 eval {
  58     require PVE::Network::SDN::Zones;
  59     $have_sdn = 1;
  60 };
  61
  62 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  63 my $OVMF = {
  64     x86_64 => [
  65         "$EDK2_FW_BASE/OVMF_CODE.fd",
  66         "$EDK2_FW_BASE/OVMF_VARS.fd"
  67     ],
  68     aarch64 => [
  69         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  70         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  71     ],
  72 };
  73
  74 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  75
  76 # Note about locking: we use flock on the config file protect
  77 # against concurent actions.
  78 # Aditionaly, we have a 'lock' setting in the config file. This
  79 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  80 # allowed when such lock is set. But you can ignore this kind of
  81 # lock with the --skiplock flag.
  82
  83 cfs_register_file('/qemu-server/',
  84                   \&parse_vm_config,
  85                   \&write_vm_config);
  86
  87 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  88     description => "Some command save/restore state from this location.",
  89     type => 'string',
  90     maxLength => 128,
  91     optional => 1,
  92 });
  93
  94 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  95         description => "Specifies the Qemu machine type.",
  96         type => 'string',
  97         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  98         maxLength => 40,
  99         optional => 1,
 100 });
 101
 102
 103 sub map_storage {
 104     my ($map, $source) = @_;
 105
 106     return $source if !defined($map);
 107
 108     return $map->{entries}->{$source}
 109         if $map->{entries} && defined($map->{entries}->{$source});
 110
 111     return $map->{default} if $map->{default};
 112
 113     # identity (fallback)
 114     return $source;
 115 }
 116
 117 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 118     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 119     type => 'string',
 120     format => 'storagepair-list',
 121     optional => 1,
 122 });
 123
 124 #no warnings 'redefine';
 125
 126 my $nodename_cache;
 127 sub nodename {
 128     $nodename_cache //= PVE::INotify::nodename();
 129     return $nodename_cache;
 130 }
 131
 132 my $watchdog_fmt = {
 133     model => {
 134         default_key => 1,
 135         type => 'string',
 136         enum => [qw(i6300esb ib700)],
 137         description => "Watchdog type to emulate.",
 138         default => 'i6300esb',
 139         optional => 1,
 140     },
 141     action => {
 142         type => 'string',
 143         enum => [qw(reset shutdown poweroff pause debug none)],
 144         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 145         optional => 1,
 146     },
 147 };
 148 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 149
 150 my $agent_fmt = {
 151     enabled => {
 152         description => "Enable/disable Qemu GuestAgent.",
 153         type => 'boolean',
 154         default => 0,
 155         default_key => 1,
 156     },
 157     fstrim_cloned_disks => {
 158         description => "Run fstrim after cloning/moving a disk.",
 159         type => 'boolean',
 160         optional => 1,
 161         default => 0
 162     },
 163     type => {
 164         description => "Select the agent type",
 165         type => 'string',
 166         default => 'virtio',
 167         optional => 1,
 168         enum => [qw(virtio isa)],
 169     },
 170 };
 171
 172 my $vga_fmt = {
 173     type => {
 174         description => "Select the VGA type.",
 175         type => 'string',
 176         default => 'std',
 177         optional => 1,
 178         default_key => 1,
 179         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 180     },
 181     memory => {
 182         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 183         type => 'integer',
 184         optional => 1,
 185         minimum => 4,
 186         maximum => 512,
 187     },
 188 };
 189
 190 my $ivshmem_fmt = {
 191     size => {
 192         type => 'integer',
 193         minimum => 1,
 194         description => "The size of the file in MB.",
 195     },
 196     name => {
 197         type => 'string',
 198         pattern => '[a-zA-Z0-9\-]+',
 199         optional => 1,
 200         format_description => 'string',
 201         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 202     },
 203 };
 204
 205 my $audio_fmt = {
 206     device => {
 207         type => 'string',
 208         enum => [qw(ich9-intel-hda intel-hda AC97)],
 209         description =>  "Configure an audio device."
 210     },
 211     driver =>  {
 212         type => 'string',
 213         enum => ['spice'],
 214         default => 'spice',
 215         optional => 1,
 216         description => "Driver backend for the audio device."
 217     },
 218 };
 219
 220 my $spice_enhancements_fmt = {
 221     foldersharing => {
 222         type => 'boolean',
 223         optional => 1,
 224         default => '0',
 225         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 226     },
 227     videostreaming =>  {
 228         type => 'string',
 229         enum => ['off', 'all', 'filter'],
 230         default => 'off',
 231         optional => 1,
 232         description => "Enable video streaming. Uses compression for detected video streams."
 233     },
 234 };
 235
 236 my $rng_fmt = {
 237     source => {
 238         type => 'string',
 239         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 240         default_key => 1,
 241         description => "The file on the host to gather entropy from. In most"
 242                      . " cases /dev/urandom should be preferred over /dev/random"
 243                      . " to avoid entropy-starvation issues on the host. Using"
 244                      . " urandom does *not* decrease security in any meaningful"
 245                      . " way, as it's still seeded from real entropy, and the"
 246                      . " bytes provided will most likely be mixed with real"
 247                      . " entropy on the guest as well. /dev/hwrng can be used"
 248                      . " to pass through a hardware RNG from the host.",
 249     },
 250     max_bytes => {
 251         type => 'integer',
 252         description => "Maximum bytes of entropy injected into the guest every"
 253                      . " 'period' milliseconds. Prefer a lower value when using"
 254                      . " /dev/random as source. Use 0 to disable limiting"
 255                      . " (potentially dangerous!).",
 256         optional => 1,
 257
 258         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 259         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 260         # of overwhelming the host, provided we're reading from /dev/urandom
 261         default => 1024,
 262     },
 263     period => {
 264         type => 'integer',
 265         description => "Every 'period' milliseconds the entropy-injection quota"
 266                      . " is reset, allowing the guest to retrieve another"
 267                      . " 'max_bytes' of entropy.",
 268         optional => 1,
 269         default => 1000,
 270     },
 271 };
 272
 273 my $confdesc = {
 274     onboot => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Specifies whether a VM will be started during system bootup.",
 278         default => 0,
 279     },
 280     autostart => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Automatic restart after crash (currently ignored).",
 284         default => 0,
 285     },
 286     hotplug => {
 287         optional => 1,
 288         type => 'string', format => 'pve-hotplug-features',
 289         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 290         default => 'network,disk,usb',
 291     },
 292     reboot => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 296         default => 1,
 297     },
 298     lock => {
 299         optional => 1,
 300         type => 'string',
 301         description => "Lock/unlock the VM.",
 302         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 303     },
 304     cpulimit => {
 305         optional => 1,
 306         type => 'number',
 307         description => "Limit of CPU usage.",
 308         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 309         minimum => 0,
 310         maximum => 128,
 311         default => 0,
 312     },
 313     cpuunits => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "CPU weight for a VM.",
 317         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 318         minimum => 2,
 319         maximum => 262144,
 320         default => 1024,
 321     },
 322     memory => {
 323         optional => 1,
 324         type => 'integer',
 325         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 326         minimum => 16,
 327         default => 512,
 328     },
 329     balloon => {
 330         optional => 1,
 331         type => 'integer',
 332         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 333         minimum => 0,
 334     },
 335     shares => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 339         minimum => 0,
 340         maximum => 50000,
 341         default => 1000,
 342     },
 343     keyboard => {
 344         optional => 1,
 345         type => 'string',
 346         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 347                        "It should not be necessary to set it.",
 348         enum => PVE::Tools::kvmkeymaplist(),
 349         default => undef,
 350     },
 351     name => {
 352         optional => 1,
 353         type => 'string', format => 'dns-name',
 354         description => "Set a name for the VM. Only used on the configuration web interface.",
 355     },
 356     scsihw => {
 357         optional => 1,
 358         type => 'string',
 359         description => "SCSI controller model",
 360         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 361         default => 'lsi',
 362     },
 363     description => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 367     },
 368     ostype => {
 369         optional => 1,
 370         type => 'string',
 371         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 372         description => "Specify guest operating system.",
 373         verbose_description => <<EODESC,
 374 Specify guest operating system. This is used to enable special
 375 optimization/features for specific operating systems:
 376
 377 [horizontal]
 378 other;; unspecified OS
 379 wxp;; Microsoft Windows XP
 380 w2k;; Microsoft Windows 2000
 381 w2k3;; Microsoft Windows 2003
 382 w2k8;; Microsoft Windows 2008
 383 wvista;; Microsoft Windows Vista
 384 win7;; Microsoft Windows 7
 385 win8;; Microsoft Windows 8/2012/2012r2
 386 win10;; Microsoft Windows 10/2016
 387 l24;; Linux 2.4 Kernel
 388 l26;; Linux 2.6 - 5.X Kernel
 389 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 390 EODESC
 391     },
 392     boot => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-boot',
 395         description => "Specify guest boot order. Use with 'order=', usage with"
 396                      . " no key or 'legacy=' is deprecated.",
 397     },
 398     bootdisk => {
 399         optional => 1,
 400         type => 'string', format => 'pve-qm-bootdisk',
 401         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 402         pattern => '(ide|sata|scsi|virtio)\d+',
 403     },
 404     smp => {
 405         optional => 1,
 406         type => 'integer',
 407         description => "The number of CPUs. Please use option -sockets instead.",
 408         minimum => 1,
 409         default => 1,
 410     },
 411     sockets => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPU sockets.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     cores => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of cores per socket.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     numa => {
 426         optional => 1,
 427         type => 'boolean',
 428         description => "Enable/disable NUMA.",
 429         default => 0,
 430     },
 431     hugepages => {
 432         optional => 1,
 433         type => 'string',
 434         description => "Enable/disable hugepages memory.",
 435         enum => [qw(any 2 1024)],
 436     },
 437     keephugepages => {
 438         optional => 1,
 439         type => 'boolean',
 440         default => 0,
 441         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 442             ." after VM shutdown and can be used for subsequent starts.",
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype"
 479             ." indicates a Microsoft OS.",
 480     },
 481     freeze => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 485     },
 486     vga => {
 487         optional => 1,
 488         type => 'string', format => $vga_fmt,
 489         description => "Configure the VGA hardware.",
 490         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 491             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 492             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 493             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 494             ." display server. For win* OS you can select how many independent displays you want,"
 495             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 496             ." using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 503             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 504             ." else the watchdog will reset the guest (or execute the respective action specified)",
 505     },
 506     startdate => {
 507         optional => 1,
 508         type => 'string',
 509         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 510         description => "Set the initial date of the real time clock. Valid format for date are:"
 511             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 540             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 541             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 542             ." may consider disabling this to save some context switches. This is turned off by"
 543             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 544     },
 545     migrate_speed => {
 546         optional => 1,
 547         type => 'integer',
 548         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 549         minimum => 0,
 550         default => 0,
 551     },
 552     migrate_downtime => {
 553         optional => 1,
 554         type => 'number',
 555         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 556         minimum => 0,
 557         default => 0.1,
 558     },
 559     cdrom => {
 560         optional => 1,
 561         type => 'string', format => 'pve-qm-ide',
 562         typetext => '<volume>',
 563         description => "This is an alias for option -ide2",
 564     },
 565     cpu => {
 566         optional => 1,
 567         description => "Emulated CPU type.",
 568         type => 'string',
 569         format => 'pve-vm-cpu-conf',
 570     },
 571     parent => get_standard_option('pve-snapshot-name', {
 572         optional => 1,
 573         description => "Parent snapshot name. This is used internally, and should not be modified.",
 574     }),
 575     snaptime => {
 576         optional => 1,
 577         description => "Timestamp for snapshots.",
 578         type => 'integer',
 579         minimum => 0,
 580     },
 581     vmstate => {
 582         optional => 1,
 583         type => 'string', format => 'pve-volume-id',
 584         description => "Reference to a volume which stores the VM state. This is used internally"
 585             ." for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 593             ." for snapshots.",
 594     }),
 595     runningcpu => {
 596         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 597             ." internally for snapshots.",
 598         optional => 1,
 599         type => 'string',
 600         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 601         format_description => 'QEMU -cpu parameter'
 602     },
 603     machine => get_standard_option('pve-qemu-machine'),
 604     arch => {
 605         description => "Virtual processor architecture. Defaults to the host.",
 606         optional => 1,
 607         type => 'string',
 608         enum => [qw(x86_64 aarch64)],
 609     },
 610     smbios1 => {
 611         description => "Specify SMBIOS type 1 fields.",
 612         type => 'string', format => 'pve-qm-smbios1',
 613         maxLength => 512,
 614         optional => 1,
 615     },
 616     protection => {
 617         optional => 1,
 618         type => 'boolean',
 619         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 620             ." remove disk operations.",
 621         default => 0,
 622     },
 623     bios => {
 624         optional => 1,
 625         type => 'string',
 626         enum => [ qw(seabios ovmf) ],
 627         description => "Select BIOS implementation.",
 628         default => 'seabios',
 629     },
 630     vmgenid => {
 631         type => 'string',
 632         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 633         format_description => 'UUID',
 634         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 635             ." to disable explicitly.",
 636         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 637             ." value identifier to the guest OS. This allows to notify the guest operating system"
 638             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 639             ." execution or creation from a template). The guest operating system notices the"
 640             ." change, and is then able to react as appropriate by marking its copies of"
 641             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 642             ."Note that auto-creation only works when done through API/CLI create or update methods"
 643             .", but not when manually editing the config file.",
 644         default => "1 (autogenerated)",
 645         optional => 1,
 646     },
 647     hookscript => {
 648         type => 'string',
 649         format => 'pve-volume-id',
 650         optional => 1,
 651         description => "Script that will be executed during various steps in the vms lifetime.",
 652     },
 653     ivshmem => {
 654         type => 'string',
 655         format => $ivshmem_fmt,
 656         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 657             ." the host.",
 658         optional => 1,
 659     },
 660     audio0 => {
 661         type => 'string',
 662         format => $audio_fmt,
 663         description => "Configure a audio device, useful in combination with QXL/Spice.",
 664         optional => 1
 665     },
 666     spice_enhancements => {
 667         type => 'string',
 668         format => $spice_enhancements_fmt,
 669         description => "Configure additional enhancements for SPICE.",
 670         optional => 1
 671     },
 672     tags => {
 673         type => 'string', format => 'pve-tag-list',
 674         description => 'Tags of the VM. This is only meta information.',
 675         optional => 1,
 676     },
 677     rng0 => {
 678         type => 'string',
 679         format => $rng_fmt,
 680         description => "Configure a VirtIO-based Random Number Generator.",
 681         optional => 1,
 682     },
 683 };
 684
 685 my $cicustom_fmt = {
 686     meta => {
 687         type => 'string',
 688         optional => 1,
 689         description => 'Specify a custom file containing all meta data passed to the VM via"
 690             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 691         format => 'pve-volume-id',
 692         format_description => 'volume',
 693     },
 694     network => {
 695         type => 'string',
 696         optional => 1,
 697         description => 'Specify a custom file containing all network data passed to the VM via'
 698             .' cloud-init.',
 699         format => 'pve-volume-id',
 700         format_description => 'volume',
 701     },
 702     user => {
 703         type => 'string',
 704         optional => 1,
 705         description => 'Specify a custom file containing all user data passed to the VM via'
 706             .' cloud-init.',
 707         format => 'pve-volume-id',
 708         format_description => 'volume',
 709     },
 710 };
 711 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 712
 713 my $confdesc_cloudinit = {
 714     citype => {
 715         optional => 1,
 716         type => 'string',
 717         description => 'Specifies the cloud-init configuration format. The default depends on the'
 718             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 719             .' and `configdrive2` for windows.',
 720         enum => ['configdrive2', 'nocloud'],
 721     },
 722     ciuser => {
 723         optional => 1,
 724         type => 'string',
 725         description => "cloud-init: User name to change ssh keys and password for instead of the"
 726             ." image's configured default user.",
 727     },
 728     cipassword => {
 729         optional => 1,
 730         type => 'string',
 731         description => 'cloud-init: Password to assign the user. Using this is generally not'
 732             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 733             .' support hashed passwords.',
 734     },
 735     cicustom => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'cloud-init: Specify custom files to replace the automatically generated'
 739             .' ones at start.',
 740         format => 'pve-qm-cicustom',
 741     },
 742     searchdomain => {
 743         optional => 1,
 744         type => 'string',
 745         description => "cloud-init: Sets DNS search domains for a container. Create will'
 746             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 747             .' are set.",
 748     },
 749     nameserver => {
 750         optional => 1,
 751         type => 'string', format => 'address-list',
 752         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 753             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 754             .' are set.",
 755     },
 756     sshkeys => {
 757         optional => 1,
 758         type => 'string',
 759         format => 'urlencoded',
 760         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 761     },
 762 };
 763
 764 # what about other qemu settings ?
 765 #cpu => 'string',
 766 #machine => 'string',
 767 #fda => 'file',
 768 #fdb => 'file',
 769 #mtdblock => 'file',
 770 #sd => 'file',
 771 #pflash => 'file',
 772 #snapshot => 'bool',
 773 #bootp => 'file',
 774 ##tftp => 'dir',
 775 ##smb => 'dir',
 776 #kernel => 'file',
 777 #append => 'string',
 778 #initrd => 'file',
 779 ##soundhw => 'string',
 780
 781 while (my ($k, $v) = each %$confdesc) {
 782     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 783 }
 784
 785 my $MAX_USB_DEVICES = 5;
 786 my $MAX_NETS = 32;
 787 my $MAX_SERIAL_PORTS = 4;
 788 my $MAX_PARALLEL_PORTS = 3;
 789 my $MAX_NUMA = 8;
 790
 791 my $numa_fmt = {
 792     cpus => {
 793         type => "string",
 794         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 795         description => "CPUs accessing this NUMA node.",
 796         format_description => "id[-id];...",
 797     },
 798     memory => {
 799         type => "number",
 800         description => "Amount of memory this NUMA node provides.",
 801         optional => 1,
 802     },
 803     hostnodes => {
 804         type => "string",
 805         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 806         description => "Host NUMA nodes to use.",
 807         format_description => "id[-id];...",
 808         optional => 1,
 809     },
 810     policy => {
 811         type => 'string',
 812         enum => [qw(preferred bind interleave)],
 813         description => "NUMA allocation policy.",
 814         optional => 1,
 815     },
 816 };
 817 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 818 my $numadesc = {
 819     optional => 1,
 820     type => 'string', format => $numa_fmt,
 821     description => "NUMA topology.",
 822 };
 823 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 824
 825 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 826     $confdesc->{"numa$i"} = $numadesc;
 827 }
 828
 829 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 830                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 831                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 832 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 833
 834 my $net_fmt_bridge_descr = <<__EOD__;
 835 Bridge to attach the network device to. The Proxmox VE standard bridge
 836 is called 'vmbr0'.
 837
 838 If you do not specify a bridge, we create a kvm user (NATed) network
 839 device, which provides DHCP and DNS services. The following addresses
 840 are used:
 841
 842  10.0.2.2   Gateway
 843  10.0.2.3   DNS Server
 844  10.0.2.4   SMB Server
 845
 846 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 847 __EOD__
 848
 849 my $net_fmt = {
 850     macaddr  => get_standard_option('mac-addr', {
 851         description => "MAC address. That address must be unique withing your network. This is"
 852             ." automatically generated if not specified.",
 853     }),
 854     model => {
 855         type => 'string',
 856         description => "Network Card Model. The 'virtio' model provides the best performance with"
 857             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 858             ." best to use 'e1000'.",
 859         enum => $nic_model_list,
 860         default_key => 1,
 861     },
 862     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 863     bridge => {
 864         type => 'string',
 865         description => $net_fmt_bridge_descr,
 866         format_description => 'bridge',
 867         pattern => '[-_.\w\d]+',
 868         optional => 1,
 869     },
 870     queues => {
 871         type => 'integer',
 872         minimum => 0, maximum => 16,
 873         description => 'Number of packet queues to be used on the device.',
 874         optional => 1,
 875     },
 876     rate => {
 877         type => 'number',
 878         minimum => 0,
 879         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 880         optional => 1,
 881     },
 882     tag => {
 883         type => 'integer',
 884         minimum => 1, maximum => 4094,
 885         description => 'VLAN tag to apply to packets on this interface.',
 886         optional => 1,
 887     },
 888     trunks => {
 889         type => 'string',
 890         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 891         description => 'VLAN trunks to pass through this interface.',
 892         format_description => 'vlanid[;vlanid...]',
 893         optional => 1,
 894     },
 895     firewall => {
 896         type => 'boolean',
 897         description => 'Whether this interface should be protected by the firewall.',
 898         optional => 1,
 899     },
 900     link_down => {
 901         type => 'boolean',
 902         description => 'Whether this interface should be disconnected (like pulling the plug).',
 903         optional => 1,
 904     },
 905     mtu => {
 906         type => 'integer',
 907         minimum => 1, maximum => 65520,
 908         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 909         optional => 1,
 910     },
 911 };
 912
 913 my $netdesc = {
 914     optional => 1,
 915     type => 'string', format => $net_fmt,
 916     description => "Specify network devices.",
 917 };
 918
 919 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 920
 921 my $ipconfig_fmt = {
 922     ip => {
 923         type => 'string',
 924         format => 'pve-ipv4-config',
 925         format_description => 'IPv4Format/CIDR',
 926         description => 'IPv4 address in CIDR format.',
 927         optional => 1,
 928         default => 'dhcp',
 929     },
 930     gw => {
 931         type => 'string',
 932         format => 'ipv4',
 933         format_description => 'GatewayIPv4',
 934         description => 'Default gateway for IPv4 traffic.',
 935         optional => 1,
 936         requires => 'ip',
 937     },
 938     ip6 => {
 939         type => 'string',
 940         format => 'pve-ipv6-config',
 941         format_description => 'IPv6Format/CIDR',
 942         description => 'IPv6 address in CIDR format.',
 943         optional => 1,
 944         default => 'dhcp',
 945     },
 946     gw6 => {
 947         type => 'string',
 948         format => 'ipv6',
 949         format_description => 'GatewayIPv6',
 950         description => 'Default gateway for IPv6 traffic.',
 951         optional => 1,
 952         requires => 'ip6',
 953     },
 954 };
 955 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 956 my $ipconfigdesc = {
 957     optional => 1,
 958     type => 'string', format => 'pve-qm-ipconfig',
 959     description => <<'EODESCR',
 960 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 961
 962 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 963
 964 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 965 gateway should be provided.
 966 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 967
 968 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 969 dhcp on IPv4.
 970 EODESCR
 971 };
 972 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 973
 974 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 975     $confdesc->{"net$i"} = $netdesc;
 976     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 977 }
 978
 979 foreach my $key (keys %$confdesc_cloudinit) {
 980     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 981 }
 982
 983 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 984 sub verify_volume_id_or_qm_path {
 985     my ($volid, $noerr) = @_;
 986
 987     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 988         return $volid;
 989     }
 990
 991     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 992     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 993     if ($@) {
 994         return if $noerr;
 995         die $@;
 996     }
 997     return $volid;
 998 }
 999
1000 my $usb_fmt = {
1001     host => {
1002         default_key => 1,
1003         type => 'string', format => 'pve-qm-usb-device',
1004         format_description => 'HOSTUSBDEVICE|spice',
1005         description => <<EODESCR,
1006 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1007
1008  'bus-port(.port)*' (decimal numbers) or
1009  'vendor_id:product_id' (hexadeciaml numbers) or
1010  'spice'
1011
1012 You can use the 'lsusb -t' command to list existing usb devices.
1013
1014 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1015 machines - use with special care.
1016
1017 The value 'spice' can be used to add a usb redirection devices for spice.
1018 EODESCR
1019     },
1020     usb3 => {
1021         optional => 1,
1022         type => 'boolean',
1023         description => "Specifies whether if given host option is a USB3 device or port.",
1024         default => 0,
1025     },
1026 };
1027
1028 my $usbdesc = {
1029     optional => 1,
1030     type => 'string', format => $usb_fmt,
1031     description => "Configure an USB device (n is 0 to 4).",
1032 };
1033 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1034
1035 my $serialdesc = {
1036         optional => 1,
1037         type => 'string',
1038         pattern => '(/dev/.+|socket)',
1039         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1040         verbose_description =>  <<EODESCR,
1041 Create a serial device inside the VM (n is 0 to 3), and pass through a
1042 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1043 host side (use 'qm terminal' to open a terminal connection).
1044
1045 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1046 use with special care.
1047
1048 CAUTION: Experimental! User reported problems with this option.
1049 EODESCR
1050 };
1051
1052 my $paralleldesc= {
1053         optional => 1,
1054         type => 'string',
1055         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1056         description =>  "Map host parallel devices (n is 0 to 2).",
1057         verbose_description =>  <<EODESCR,
1058 Map host parallel devices (n is 0 to 2).
1059
1060 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1061 machines - use with special care.
1062
1063 CAUTION: Experimental! User reported problems with this option.
1064 EODESCR
1065 };
1066
1067 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1068     $confdesc->{"parallel$i"} = $paralleldesc;
1069 }
1070
1071 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1072     $confdesc->{"serial$i"} = $serialdesc;
1073 }
1074
1075 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1076     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1077 }
1078
1079 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1080     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1081 }
1082
1083 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1084     $confdesc->{"usb$i"} = $usbdesc;
1085 }
1086
1087 my $boot_fmt = {
1088     legacy => {
1089         optional => 1,
1090         default_key => 1,
1091         type => 'string',
1092         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1093                      . " Deprecated, use 'order=' instead.",
1094         pattern => '[acdn]{1,4}',
1095         format_description => "[acdn]{1,4}",
1096
1097         # note: this is also the fallback if boot: is not given at all
1098         default => 'cdn',
1099     },
1100     order => {
1101         optional => 1,
1102         type => 'string',
1103         format => 'pve-qm-bootdev-list',
1104         format_description => "device[;device...]",
1105         description => <<EODESC,
1106 The guest will attempt to boot from devices in the order they appear here.
1107
1108 Disks, optical drives and passed-through storage USB devices will be directly
1109 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1110 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1111
1112 Note that only devices in this list will be marked as bootable and thus loaded
1113 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1114 (e.g. software-raid), you need to specify all of them here.
1115
1116 Overrides the deprecated 'legacy=[acdn]*' value when given.
1117 EODESC
1118     },
1119 };
1120 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1121
1122 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1123 sub verify_bootdev {
1124     my ($dev, $noerr) = @_;
1125
1126     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1127
1128     my $check = sub {
1129         my ($base) = @_;
1130         return 0 if $dev !~ m/^$base\d+$/;
1131         return 0 if !$confdesc->{$dev};
1132         return 1;
1133     };
1134
1135     return $dev if $check->("net");
1136     return $dev if $check->("usb");
1137     return $dev if $check->("hostpci");
1138
1139     return if $noerr;
1140     die "invalid boot device '$dev'\n";
1141 }
1142
1143 sub print_bootorder {
1144     my ($devs) = @_;
1145     return "" if !@$devs;
1146     my $data = { order => join(';', @$devs) };
1147     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1148 }
1149
1150 my $kvm_api_version = 0;
1151
1152 sub kvm_version {
1153     return $kvm_api_version if $kvm_api_version;
1154
1155     open my $fh, '<', '/dev/kvm' or return;
1156
1157     # 0xae00 => KVM_GET_API_VERSION
1158     $kvm_api_version = ioctl($fh, 0xae00, 0);
1159     close($fh);
1160
1161     return $kvm_api_version;
1162 }
1163
1164 my $kvm_user_version = {};
1165 my $kvm_mtime = {};
1166
1167 sub kvm_user_version {
1168     my ($binary) = @_;
1169
1170     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1171     my $st = stat($binary);
1172
1173     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1174     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1175         $cachedmtime == $st->mtime;
1176
1177     $kvm_user_version->{$binary} = 'unknown';
1178     $kvm_mtime->{$binary} = $st->mtime;
1179
1180     my $code = sub {
1181         my $line = shift;
1182         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1183             $kvm_user_version->{$binary} = $2;
1184         }
1185     };
1186
1187     eval { run_command([$binary, '--version'], outfunc => $code); };
1188     warn $@ if $@;
1189
1190     return $kvm_user_version->{$binary};
1191
1192 }
1193 my sub extract_version {
1194     my ($machine_type, $version) = @_;
1195     $version = kvm_user_version() if !defined($version);
1196     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1197 }
1198
1199 sub kernel_has_vhost_net {
1200     return -c '/dev/vhost-net';
1201 }
1202
1203 sub option_exists {
1204     my $key = shift;
1205     return defined($confdesc->{$key});
1206 }
1207
1208 my $cdrom_path;
1209 sub get_cdrom_path {
1210
1211     return  $cdrom_path if $cdrom_path;
1212
1213     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1214     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1215     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1216 }
1217
1218 sub get_iso_path {
1219     my ($storecfg, $vmid, $cdrom) = @_;
1220
1221     if ($cdrom eq 'cdrom') {
1222         return get_cdrom_path();
1223     } elsif ($cdrom eq 'none') {
1224         return '';
1225     } elsif ($cdrom =~ m|^/|) {
1226         return $cdrom;
1227     } else {
1228         return PVE::Storage::path($storecfg, $cdrom);
1229     }
1230 }
1231
1232 # try to convert old style file names to volume IDs
1233 sub filename_to_volume_id {
1234     my ($vmid, $file, $media) = @_;
1235
1236      if (!($file eq 'none' || $file eq 'cdrom' ||
1237           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1238
1239         return if $file =~ m|/|;
1240
1241         if ($media && $media eq 'cdrom') {
1242             $file = "local:iso/$file";
1243         } else {
1244             $file = "local:$vmid/$file";
1245         }
1246     }
1247
1248     return $file;
1249 }
1250
1251 sub verify_media_type {
1252     my ($opt, $vtype, $media) = @_;
1253
1254     return if !$media;
1255
1256     my $etype;
1257     if ($media eq 'disk') {
1258         $etype = 'images';
1259     } elsif ($media eq 'cdrom') {
1260         $etype = 'iso';
1261     } else {
1262         die "internal error";
1263     }
1264
1265     return if ($vtype eq $etype);
1266
1267     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1268 }
1269
1270 sub cleanup_drive_path {
1271     my ($opt, $storecfg, $drive) = @_;
1272
1273     # try to convert filesystem paths to volume IDs
1274
1275     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1276         ($drive->{file} !~ m|^/dev/.+|) &&
1277         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1278         ($drive->{file} !~ m/^\d+$/)) {
1279         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1280         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1281             if !$vtype;
1282         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1283         verify_media_type($opt, $vtype, $drive->{media});
1284         $drive->{file} = $volid;
1285     }
1286
1287     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1288 }
1289
1290 sub parse_hotplug_features {
1291     my ($data) = @_;
1292
1293     my $res = {};
1294
1295     return $res if $data eq '0';
1296
1297     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1298
1299     foreach my $feature (PVE::Tools::split_list($data)) {
1300         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1301             $res->{$1} = 1;
1302         } else {
1303             die "invalid hotplug feature '$feature'\n";
1304         }
1305     }
1306     return $res;
1307 }
1308
1309 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1310 sub pve_verify_hotplug_features {
1311     my ($value, $noerr) = @_;
1312
1313     return $value if parse_hotplug_features($value);
1314
1315     return if $noerr;
1316
1317     die "unable to parse hotplug option\n";
1318 }
1319
1320 sub scsi_inquiry {
1321     my($fh, $noerr) = @_;
1322
1323     my $SG_IO = 0x2285;
1324     my $SG_GET_VERSION_NUM = 0x2282;
1325
1326     my $versionbuf = "\x00" x 8;
1327     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1328     if (!$ret) {
1329         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1330         return;
1331     }
1332     my $version = unpack("I", $versionbuf);
1333     if ($version < 30000) {
1334         die "scsi generic interface too old\n"  if !$noerr;
1335         return;
1336     }
1337
1338     my $buf = "\x00" x 36;
1339     my $sensebuf = "\x00" x 8;
1340     my $cmd = pack("C x3 C x1", 0x12, 36);
1341
1342     # see /usr/include/scsi/sg.h
1343     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1344
1345     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1346                       length($sensebuf), 0, length($buf), $buf,
1347                       $cmd, $sensebuf, 6000);
1348
1349     $ret = ioctl($fh, $SG_IO, $packet);
1350     if (!$ret) {
1351         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1352         return;
1353     }
1354
1355     my @res = unpack($sg_io_hdr_t, $packet);
1356     if ($res[17] || $res[18]) {
1357         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1358         return;
1359     }
1360
1361     my $res = {};
1362     (my $byte0, my $byte1, $res->{vendor},
1363      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1364
1365     $res->{removable} = $byte1 & 128 ? 1 : 0;
1366     $res->{type} = $byte0 & 31;
1367
1368     return $res;
1369 }
1370
1371 sub path_is_scsi {
1372     my ($path) = @_;
1373
1374     my $fh = IO::File->new("+<$path") || return;
1375     my $res = scsi_inquiry($fh, 1);
1376     close($fh);
1377
1378     return $res;
1379 }
1380
1381 sub print_tabletdevice_full {
1382     my ($conf, $arch) = @_;
1383
1384     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1385
1386     # we use uhci for old VMs because tablet driver was buggy in older qemu
1387     my $usbbus;
1388     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1389         $usbbus = 'ehci';
1390     } else {
1391         $usbbus = 'uhci';
1392     }
1393
1394     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1395 }
1396
1397 sub print_keyboarddevice_full {
1398     my ($conf, $arch, $machine) = @_;
1399
1400     return if $arch ne 'aarch64';
1401
1402     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1403 }
1404
1405 sub print_drivedevice_full {
1406     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1407
1408     my $device = '';
1409     my $maxdev = 0;
1410
1411     my $drive_id = "$drive->{interface}$drive->{index}";
1412     if ($drive->{interface} eq 'virtio') {
1413         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1414         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1415         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1416     } elsif ($drive->{interface} eq 'scsi') {
1417
1418         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1419         my $unit = $drive->{index} % $maxdev;
1420         my $devicetype = 'hd';
1421         my $path = '';
1422         if (drive_is_cdrom($drive)) {
1423             $devicetype = 'cd';
1424         } else {
1425             if ($drive->{file} =~ m|^/|) {
1426                 $path = $drive->{file};
1427                 if (my $info = path_is_scsi($path)) {
1428                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1429                         $devicetype = 'block';
1430                     } elsif ($info->{type} == 1) { # tape
1431                         $devicetype = 'generic';
1432                     }
1433                 }
1434             } else {
1435                  $path = PVE::Storage::path($storecfg, $drive->{file});
1436             }
1437
1438             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1439             my $version = extract_version($machine_type, kvm_user_version());
1440             if ($path =~ m/^iscsi\:\/\// &&
1441                !min_version($version, 4, 1)) {
1442                 $devicetype = 'generic';
1443             }
1444         }
1445
1446         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1447             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1448         } else {
1449             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1450                 .",lun=$drive->{index}";
1451         }
1452         $device .= ",drive=drive-$drive_id,id=$drive_id";
1453
1454         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1455             $device .= ",rotation_rate=1";
1456         }
1457         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1458
1459     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1460         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1461         my $controller = int($drive->{index} / $maxdev);
1462         my $unit = $drive->{index} % $maxdev;
1463         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1464
1465         $device = "ide-$devicetype";
1466         if ($drive->{interface} eq 'ide') {
1467             $device .= ",bus=ide.$controller,unit=$unit";
1468         } else {
1469             $device .= ",bus=ahci$controller.$unit";
1470         }
1471         $device .= ",drive=drive-$drive_id,id=$drive_id";
1472
1473         if ($devicetype eq 'hd') {
1474             if (my $model = $drive->{model}) {
1475                 $model = URI::Escape::uri_unescape($model);
1476                 $device .= ",model=$model";
1477             }
1478             if ($drive->{ssd}) {
1479                 $device .= ",rotation_rate=1";
1480             }
1481         }
1482         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1483     } elsif ($drive->{interface} eq 'usb') {
1484         die "implement me";
1485         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1486     } else {
1487         die "unsupported interface type";
1488     }
1489
1490     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1491
1492     if (my $serial = $drive->{serial}) {
1493         $serial = URI::Escape::uri_unescape($serial);
1494         $device .= ",serial=$serial";
1495     }
1496
1497
1498     return $device;
1499 }
1500
1501 sub get_initiator_name {
1502     my $initiator;
1503
1504     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1505     while (defined(my $line = <$fh>)) {
1506         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1507         $initiator = $1;
1508         last;
1509     }
1510     $fh->close();
1511
1512     return $initiator;
1513 }
1514
1515 sub print_drive_commandline_full {
1516     my ($storecfg, $vmid, $drive) = @_;
1517
1518     my $path;
1519     my $volid = $drive->{file};
1520     my $format;
1521
1522     if (drive_is_cdrom($drive)) {
1523         $path = get_iso_path($storecfg, $vmid, $volid);
1524     } else {
1525         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1526         if ($storeid) {
1527             $path = PVE::Storage::path($storecfg, $volid);
1528             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1529             $format = qemu_img_format($scfg, $volname);
1530         } else {
1531             $path = $volid;
1532             $format = "raw";
1533         }
1534    }
1535
1536     my $opts = '';
1537     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1538     foreach my $o (@qemu_drive_options) {
1539         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1540     }
1541
1542     # snapshot only accepts on|off
1543     if (defined($drive->{snapshot})) {
1544         my $v = $drive->{snapshot} ? 'on' : 'off';
1545         $opts .= ",snapshot=$v";
1546     }
1547
1548     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1549         my ($dir, $qmpname) = @$type;
1550         if (my $v = $drive->{"mbps$dir"}) {
1551             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1552         }
1553         if (my $v = $drive->{"mbps${dir}_max"}) {
1554             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1555         }
1556         if (my $v = $drive->{"bps${dir}_max_length"}) {
1557             $opts .= ",throttling.bps$qmpname-max-length=$v";
1558         }
1559         if (my $v = $drive->{"iops${dir}"}) {
1560             $opts .= ",throttling.iops$qmpname=$v";
1561         }
1562         if (my $v = $drive->{"iops${dir}_max"}) {
1563             $opts .= ",throttling.iops$qmpname-max=$v";
1564         }
1565         if (my $v = $drive->{"iops${dir}_max_length"}) {
1566             $opts .= ",throttling.iops$qmpname-max-length=$v";
1567         }
1568     }
1569
1570     $opts .= ",format=$format" if $format && !$drive->{format};
1571
1572     my $cache_direct = 0;
1573
1574     if (my $cache = $drive->{cache}) {
1575         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1576     } elsif (!drive_is_cdrom($drive)) {
1577         $opts .= ",cache=none";
1578         $cache_direct = 1;
1579     }
1580
1581     # aio native works only with O_DIRECT
1582     if (!$drive->{aio}) {
1583         if($cache_direct) {
1584             $opts .= ",aio=native";
1585         } else {
1586             $opts .= ",aio=threads";
1587         }
1588     }
1589
1590     if (!drive_is_cdrom($drive)) {
1591         my $detectzeroes;
1592         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1593             $detectzeroes = 'off';
1594         } elsif ($drive->{discard}) {
1595             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1596         } else {
1597             # This used to be our default with discard not being specified:
1598             $detectzeroes = 'on';
1599         }
1600         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1601     }
1602
1603     my $pathinfo = $path ? "file=$path," : '';
1604
1605     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1606 }
1607
1608 sub print_netdevice_full {
1609     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1610
1611     my $device = $net->{model};
1612     if ($net->{model} eq 'virtio') {
1613          $device = 'virtio-net-pci';
1614      };
1615
1616     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1617     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1618     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1619         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1620         # and out of each queue plus one config interrupt and control vector queue
1621         my $vectors = $net->{queues} * 2 + 2;
1622         $tmpstr .= ",vectors=$vectors,mq=on";
1623     }
1624     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1625
1626     if (my $mtu = $net->{mtu}) {
1627         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1628             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1629             if ($mtu == 1) {
1630                  $mtu = $bridge_mtu;
1631             } elsif ($mtu < 576) {
1632                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1633             } elsif ($mtu > $bridge_mtu) {
1634                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1635             }
1636             $tmpstr .= ",host_mtu=$mtu";
1637         } else {
1638             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1639         }
1640     }
1641
1642     if ($use_old_bios_files) {
1643         my $romfile;
1644         if ($device eq 'virtio-net-pci') {
1645             $romfile = 'pxe-virtio.rom';
1646         } elsif ($device eq 'e1000') {
1647             $romfile = 'pxe-e1000.rom';
1648         } elsif ($device eq 'ne2k') {
1649             $romfile = 'pxe-ne2k_pci.rom';
1650         } elsif ($device eq 'pcnet') {
1651             $romfile = 'pxe-pcnet.rom';
1652         } elsif ($device eq 'rtl8139') {
1653             $romfile = 'pxe-rtl8139.rom';
1654         }
1655         $tmpstr .= ",romfile=$romfile" if $romfile;
1656     }
1657
1658     return $tmpstr;
1659 }
1660
1661 sub print_netdev_full {
1662     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1663
1664     my $i = '';
1665     if ($netid =~ m/^net(\d+)$/) {
1666         $i = int($1);
1667     }
1668
1669     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1670
1671     my $ifname = "tap${vmid}i$i";
1672
1673     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1674     die "interface name '$ifname' is too long (max 15 character)\n"
1675         if length($ifname) >= 16;
1676
1677     my $vhostparam = '';
1678     if (is_native($arch)) {
1679         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1680     }
1681
1682     my $vmname = $conf->{name} || "vm$vmid";
1683
1684     my $netdev = "";
1685     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1686
1687     if ($net->{bridge}) {
1688         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1689             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1690     } else {
1691         $netdev = "type=user,id=$netid,hostname=$vmname";
1692     }
1693
1694     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1695
1696     return $netdev;
1697 }
1698
1699 my $vga_map = {
1700     'cirrus' => 'cirrus-vga',
1701     'std' => 'VGA',
1702     'vmware' => 'vmware-svga',
1703     'virtio' => 'virtio-vga',
1704 };
1705
1706 sub print_vga_device {
1707     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1708
1709     my $type = $vga_map->{$vga->{type}};
1710     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1711         $type = 'virtio-gpu';
1712     }
1713     my $vgamem_mb = $vga->{memory};
1714
1715     my $max_outputs = '';
1716     if ($qxlnum) {
1717         $type = $id ? 'qxl' : 'qxl-vga';
1718
1719         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1720             # set max outputs so linux can have up to 4 qxl displays with one device
1721             if (min_version($machine_version, 4, 1)) {
1722                 $max_outputs = ",max_outputs=4";
1723             }
1724         }
1725     }
1726
1727     die "no devicetype for $vga->{type}\n" if !$type;
1728
1729     my $memory = "";
1730     if ($vgamem_mb) {
1731         if ($vga->{type} eq 'virtio') {
1732             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1733             $memory = ",max_hostmem=$bytes";
1734         } elsif ($qxlnum) {
1735             # from https://www.spice-space.org/multiple-monitors.html
1736             $memory = ",vgamem_mb=$vga->{memory}";
1737             my $ram = $vgamem_mb * 4;
1738             my $vram = $vgamem_mb * 2;
1739             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1740         } else {
1741             $memory = ",vgamem_mb=$vga->{memory}";
1742         }
1743     } elsif ($qxlnum && $id) {
1744         $memory = ",ram_size=67108864,vram_size=33554432";
1745     }
1746
1747     my $edidoff = "";
1748     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1749         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1750     }
1751
1752     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1753     my $vgaid = "vga" . ($id // '');
1754     my $pciaddr;
1755
1756     if ($q35 && $vgaid eq 'vga') {
1757         # the first display uses pcie.0 bus on q35 machines
1758         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1759     } else {
1760         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1761     }
1762
1763     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1764 }
1765
1766 sub parse_number_sets {
1767     my ($set) = @_;
1768     my $res = [];
1769     foreach my $part (split(/;/, $set)) {
1770         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1771             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1772             push @$res, [ $1, $2 ];
1773         } else {
1774             die "invalid range: $part\n";
1775         }
1776     }
1777     return $res;
1778 }
1779
1780 sub parse_numa {
1781     my ($data) = @_;
1782
1783     my $res = parse_property_string($numa_fmt, $data);
1784     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1785     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1786     return $res;
1787 }
1788
1789 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1790 sub parse_net {
1791     my ($data) = @_;
1792
1793     my $res = eval { parse_property_string($net_fmt, $data) };
1794     if ($@) {
1795         warn $@;
1796         return;
1797     }
1798     if (!defined($res->{macaddr})) {
1799         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1800         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1801     }
1802     return $res;
1803 }
1804
1805 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1806 sub parse_ipconfig {
1807     my ($data) = @_;
1808
1809     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1810     if ($@) {
1811         warn $@;
1812         return;
1813     }
1814
1815     if ($res->{gw} && !$res->{ip}) {
1816         warn 'gateway specified without specifying an IP address';
1817         return;
1818     }
1819     if ($res->{gw6} && !$res->{ip6}) {
1820         warn 'IPv6 gateway specified without specifying an IPv6 address';
1821         return;
1822     }
1823     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1824         warn 'gateway specified together with DHCP';
1825         return;
1826     }
1827     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1828         # gw6 + auto/dhcp
1829         warn "IPv6 gateway specified together with $res->{ip6} address";
1830         return;
1831     }
1832
1833     if (!$res->{ip} && !$res->{ip6}) {
1834         return { ip => 'dhcp', ip6 => 'dhcp' };
1835     }
1836
1837     return $res;
1838 }
1839
1840 sub print_net {
1841     my $net = shift;
1842
1843     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1844 }
1845
1846 sub add_random_macs {
1847     my ($settings) = @_;
1848
1849     foreach my $opt (keys %$settings) {
1850         next if $opt !~ m/^net(\d+)$/;
1851         my $net = parse_net($settings->{$opt});
1852         next if !$net;
1853         $settings->{$opt} = print_net($net);
1854     }
1855 }
1856
1857 sub vm_is_volid_owner {
1858     my ($storecfg, $vmid, $volid) = @_;
1859
1860     if ($volid !~  m|^/|) {
1861         my ($path, $owner);
1862         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1863         if ($owner && ($owner == $vmid)) {
1864             return 1;
1865         }
1866     }
1867
1868     return;
1869 }
1870
1871 sub vmconfig_register_unused_drive {
1872     my ($storecfg, $vmid, $conf, $drive) = @_;
1873
1874     if (drive_is_cloudinit($drive)) {
1875         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1876         warn $@ if $@;
1877     } elsif (!drive_is_cdrom($drive)) {
1878         my $volid = $drive->{file};
1879         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1880             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1881         }
1882     }
1883 }
1884
1885 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1886 my $smbios1_fmt = {
1887     uuid => {
1888         type => 'string',
1889         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1890         format_description => 'UUID',
1891         description => "Set SMBIOS1 UUID.",
1892         optional => 1,
1893     },
1894     version => {
1895         type => 'string',
1896         pattern => '[A-Za-z0-9+\/]+={0,2}',
1897         format_description => 'Base64 encoded string',
1898         description => "Set SMBIOS1 version.",
1899         optional => 1,
1900     },
1901     serial => {
1902         type => 'string',
1903         pattern => '[A-Za-z0-9+\/]+={0,2}',
1904         format_description => 'Base64 encoded string',
1905         description => "Set SMBIOS1 serial number.",
1906         optional => 1,
1907     },
1908     manufacturer => {
1909         type => 'string',
1910         pattern => '[A-Za-z0-9+\/]+={0,2}',
1911         format_description => 'Base64 encoded string',
1912         description => "Set SMBIOS1 manufacturer.",
1913         optional => 1,
1914     },
1915     product => {
1916         type => 'string',
1917         pattern => '[A-Za-z0-9+\/]+={0,2}',
1918         format_description => 'Base64 encoded string',
1919         description => "Set SMBIOS1 product ID.",
1920         optional => 1,
1921     },
1922     sku => {
1923         type => 'string',
1924         pattern => '[A-Za-z0-9+\/]+={0,2}',
1925         format_description => 'Base64 encoded string',
1926         description => "Set SMBIOS1 SKU string.",
1927         optional => 1,
1928     },
1929     family => {
1930         type => 'string',
1931         pattern => '[A-Za-z0-9+\/]+={0,2}',
1932         format_description => 'Base64 encoded string',
1933         description => "Set SMBIOS1 family string.",
1934         optional => 1,
1935     },
1936     base64 => {
1937         type => 'boolean',
1938         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1939         optional => 1,
1940     },
1941 };
1942
1943 sub parse_smbios1 {
1944     my ($data) = @_;
1945
1946     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1947     warn $@ if $@;
1948     return $res;
1949 }
1950
1951 sub print_smbios1 {
1952     my ($smbios1) = @_;
1953     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1954 }
1955
1956 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1957
1958 sub parse_watchdog {
1959     my ($value) = @_;
1960
1961     return if !$value;
1962
1963     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1964     warn $@ if $@;
1965     return $res;
1966 }
1967
1968 sub parse_guest_agent {
1969     my ($conf) = @_;
1970
1971     return {} if !defined($conf->{agent});
1972
1973     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
1974     warn $@ if $@;
1975
1976     # if the agent is disabled ignore the other potentially set properties
1977     return {} if !$res->{enabled};
1978     return $res;
1979 }
1980
1981 sub get_qga_key {
1982     my ($conf, $key) = @_;
1983     return undef if !defined($conf->{agent});
1984
1985     my $agent = parse_guest_agent($conf);
1986     return $agent->{$key};
1987 }
1988
1989 sub parse_vga {
1990     my ($value) = @_;
1991
1992     return {} if !$value;
1993     my $res = eval { parse_property_string($vga_fmt, $value) };
1994     warn $@ if $@;
1995     return $res;
1996 }
1997
1998 sub parse_rng {
1999     my ($value) = @_;
2000
2001     return if !$value;
2002
2003     my $res = eval { parse_property_string($rng_fmt, $value) };
2004     warn $@ if $@;
2005     return $res;
2006 }
2007
2008 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2009 sub verify_usb_device {
2010     my ($value, $noerr) = @_;
2011
2012     return $value if parse_usb_device($value);
2013
2014     return if $noerr;
2015
2016     die "unable to parse usb device\n";
2017 }
2018
2019 # add JSON properties for create and set function
2020 sub json_config_properties {
2021     my $prop = shift;
2022
2023     foreach my $opt (keys %$confdesc) {
2024         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2025             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2026         $prop->{$opt} = $confdesc->{$opt};
2027     }
2028
2029     return $prop;
2030 }
2031
2032 # return copy of $confdesc_cloudinit to generate documentation
2033 sub cloudinit_config_properties {
2034
2035     return dclone($confdesc_cloudinit);
2036 }
2037
2038 sub check_type {
2039     my ($key, $value) = @_;
2040
2041     die "unknown setting '$key'\n" if !$confdesc->{$key};
2042
2043     my $type = $confdesc->{$key}->{type};
2044
2045     if (!defined($value)) {
2046         die "got undefined value\n";
2047     }
2048
2049     if ($value =~ m/[\n\r]/) {
2050         die "property contains a line feed\n";
2051     }
2052
2053     if ($type eq 'boolean') {
2054         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2055         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2056         die "type check ('boolean') failed - got '$value'\n";
2057     } elsif ($type eq 'integer') {
2058         return int($1) if $value =~ m/^(\d+)$/;
2059         die "type check ('integer') failed - got '$value'\n";
2060     } elsif ($type eq 'number') {
2061         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2062         die "type check ('number') failed - got '$value'\n";
2063     } elsif ($type eq 'string') {
2064         if (my $fmt = $confdesc->{$key}->{format}) {
2065             PVE::JSONSchema::check_format($fmt, $value);
2066             return $value;
2067         }
2068         $value =~ s/^\"(.*)\"$/$1/;
2069         return $value;
2070     } else {
2071         die "internal error"
2072     }
2073 }
2074
2075 sub destroy_vm {
2076     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2077
2078     my $conf = PVE::QemuConfig->load_config($vmid);
2079
2080     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2081
2082     if ($conf->{template}) {
2083         # check if any base image is still used by a linked clone
2084         PVE::QemuConfig->foreach_volume($conf, sub {
2085                 my ($ds, $drive) = @_;
2086                 return if drive_is_cdrom($drive);
2087
2088                 my $volid = $drive->{file};
2089                 return if !$volid || $volid =~ m|^/|;
2090
2091                 die "base volume '$volid' is still in use by linked cloned\n"
2092                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2093
2094         });
2095     }
2096
2097     # only remove disks owned by this VM
2098     PVE::QemuConfig->foreach_volume($conf, sub {
2099         my ($ds, $drive) = @_;
2100         return if drive_is_cdrom($drive, 1);
2101
2102         my $volid = $drive->{file};
2103         return if !$volid || $volid =~ m|^/|;
2104
2105         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2106         return if !$path || !$owner || ($owner != $vmid);
2107
2108         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2109         warn "Could not remove disk '$volid', check manually: $@" if $@;
2110     });
2111
2112     # also remove unused disk
2113     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2114     PVE::Storage::foreach_volid($vmdisks, sub {
2115         my ($volid, $sid, $volname, $d) = @_;
2116         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2117         warn $@ if $@;
2118     });
2119
2120     if (defined $replacement_conf) {
2121         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2122     } else {
2123         PVE::QemuConfig->destroy_config($vmid);
2124     }
2125 }
2126
2127 sub parse_vm_config {
2128     my ($filename, $raw) = @_;
2129
2130     return if !defined($raw);
2131
2132     my $res = {
2133         digest => Digest::SHA::sha1_hex($raw),
2134         snapshots => {},
2135         pending => {},
2136     };
2137
2138     $filename =~ m|/qemu-server/(\d+)\.conf$|
2139         || die "got strange filename '$filename'";
2140
2141     my $vmid = $1;
2142
2143     my $conf = $res;
2144     my $descr;
2145     my $section = '';
2146
2147     my @lines = split(/\n/, $raw);
2148     foreach my $line (@lines) {
2149         next if $line =~ m/^\s*$/;
2150
2151         if ($line =~ m/^\[PENDING\]\s*$/i) {
2152             $section = 'pending';
2153             if (defined($descr)) {
2154                 $descr =~ s/\s+$//;
2155                 $conf->{description} = $descr;
2156             }
2157             $descr = undef;
2158             $conf = $res->{$section} = {};
2159             next;
2160
2161         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2162             $section = $1;
2163             if (defined($descr)) {
2164                 $descr =~ s/\s+$//;
2165                 $conf->{description} = $descr;
2166             }
2167             $descr = undef;
2168             $conf = $res->{snapshots}->{$section} = {};
2169             next;
2170         }
2171
2172         if ($line =~ m/^\#(.*)\s*$/) {
2173             $descr = '' if !defined($descr);
2174             $descr .= PVE::Tools::decode_text($1) . "\n";
2175             next;
2176         }
2177
2178         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2179             $descr = '' if !defined($descr);
2180             $descr .= PVE::Tools::decode_text($2);
2181         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2182             $conf->{snapstate} = $1;
2183         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2184             my $key = $1;
2185             my $value = $2;
2186             $conf->{$key} = $value;
2187         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2188             my $value = $1;
2189             if ($section eq 'pending') {
2190                 $conf->{delete} = $value; # we parse this later
2191             } else {
2192                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2193             }
2194         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2195             my $key = $1;
2196             my $value = $2;
2197             eval { $value = check_type($key, $value); };
2198             if ($@) {
2199                 warn "vm $vmid - unable to parse value of '$key' - $@";
2200             } else {
2201                 $key = 'ide2' if $key eq 'cdrom';
2202                 my $fmt = $confdesc->{$key}->{format};
2203                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2204                     my $v = parse_drive($key, $value);
2205                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2206                         $v->{file} = $volid;
2207                         $value = print_drive($v);
2208                     } else {
2209                         warn "vm $vmid - unable to parse value of '$key'\n";
2210                         next;
2211                     }
2212                 }
2213
2214                 $conf->{$key} = $value;
2215             }
2216         }
2217     }
2218
2219     if (defined($descr)) {
2220         $descr =~ s/\s+$//;
2221         $conf->{description} = $descr;
2222     }
2223     delete $res->{snapstate}; # just to be sure
2224
2225     return $res;
2226 }
2227
2228 sub write_vm_config {
2229     my ($filename, $conf) = @_;
2230
2231     delete $conf->{snapstate}; # just to be sure
2232
2233     if ($conf->{cdrom}) {
2234         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2235         $conf->{ide2} = $conf->{cdrom};
2236         delete $conf->{cdrom};
2237     }
2238
2239     # we do not use 'smp' any longer
2240     if ($conf->{sockets}) {
2241         delete $conf->{smp};
2242     } elsif ($conf->{smp}) {
2243         $conf->{sockets} = $conf->{smp};
2244         delete $conf->{cores};
2245         delete $conf->{smp};
2246     }
2247
2248     my $used_volids = {};
2249
2250     my $cleanup_config = sub {
2251         my ($cref, $pending, $snapname) = @_;
2252
2253         foreach my $key (keys %$cref) {
2254             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2255                 $key eq 'snapstate' || $key eq 'pending';
2256             my $value = $cref->{$key};
2257             if ($key eq 'delete') {
2258                 die "propertry 'delete' is only allowed in [PENDING]\n"
2259                     if !$pending;
2260                 # fixme: check syntax?
2261                 next;
2262             }
2263             eval { $value = check_type($key, $value); };
2264             die "unable to parse value of '$key' - $@" if $@;
2265
2266             $cref->{$key} = $value;
2267
2268             if (!$snapname && is_valid_drivename($key)) {
2269                 my $drive = parse_drive($key, $value);
2270                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2271             }
2272         }
2273     };
2274
2275     &$cleanup_config($conf);
2276
2277     &$cleanup_config($conf->{pending}, 1);
2278
2279     foreach my $snapname (keys %{$conf->{snapshots}}) {
2280         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2281         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2282     }
2283
2284     # remove 'unusedX' settings if we re-add a volume
2285     foreach my $key (keys %$conf) {
2286         my $value = $conf->{$key};
2287         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2288             delete $conf->{$key};
2289         }
2290     }
2291
2292     my $generate_raw_config = sub {
2293         my ($conf, $pending) = @_;
2294
2295         my $raw = '';
2296
2297         # add description as comment to top of file
2298         if (defined(my $descr = $conf->{description})) {
2299             if ($descr) {
2300                 foreach my $cl (split(/\n/, $descr)) {
2301                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2302                 }
2303             } else {
2304                 $raw .= "#\n" if $pending;
2305             }
2306         }
2307
2308         foreach my $key (sort keys %$conf) {
2309             next if $key =~ /^(digest|description|pending|snapshots)$/;
2310             $raw .= "$key: $conf->{$key}\n";
2311         }
2312         return $raw;
2313     };
2314
2315     my $raw = &$generate_raw_config($conf);
2316
2317     if (scalar(keys %{$conf->{pending}})){
2318         $raw .= "\n[PENDING]\n";
2319         $raw .= &$generate_raw_config($conf->{pending}, 1);
2320     }
2321
2322     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2323         $raw .= "\n[$snapname]\n";
2324         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2325     }
2326
2327     return $raw;
2328 }
2329
2330 sub load_defaults {
2331
2332     my $res = {};
2333
2334     # we use static defaults from our JSON schema configuration
2335     foreach my $key (keys %$confdesc) {
2336         if (defined(my $default = $confdesc->{$key}->{default})) {
2337             $res->{$key} = $default;
2338         }
2339     }
2340
2341     return $res;
2342 }
2343
2344 sub config_list {
2345     my $vmlist = PVE::Cluster::get_vmlist();
2346     my $res = {};
2347     return $res if !$vmlist || !$vmlist->{ids};
2348     my $ids = $vmlist->{ids};
2349     my $nodename = nodename();
2350
2351     foreach my $vmid (keys %$ids) {
2352         my $d = $ids->{$vmid};
2353         next if !$d->{node} || $d->{node} ne $nodename;
2354         next if !$d->{type} || $d->{type} ne 'qemu';
2355         $res->{$vmid}->{exists} = 1;
2356     }
2357     return $res;
2358 }
2359
2360 # test if VM uses local resources (to prevent migration)
2361 sub check_local_resources {
2362     my ($conf, $noerr) = @_;
2363
2364     my @loc_res = ();
2365
2366     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2367     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2368
2369     push @loc_res, "ivshmem" if $conf->{ivshmem};
2370
2371     foreach my $k (keys %$conf) {
2372         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2373         # sockets are safe: they will recreated be on the target side post-migrate
2374         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2375         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2376     }
2377
2378     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2379
2380     return \@loc_res;
2381 }
2382
2383 # check if used storages are available on all nodes (use by migrate)
2384 sub check_storage_availability {
2385     my ($storecfg, $conf, $node) = @_;
2386
2387     PVE::QemuConfig->foreach_volume($conf, sub {
2388         my ($ds, $drive) = @_;
2389
2390         my $volid = $drive->{file};
2391         return if !$volid;
2392
2393         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2394         return if !$sid;
2395
2396         # check if storage is available on both nodes
2397         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2398         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2399    });
2400 }
2401
2402 # list nodes where all VM images are available (used by has_feature API)
2403 sub shared_nodes {
2404     my ($conf, $storecfg) = @_;
2405
2406     my $nodelist = PVE::Cluster::get_nodelist();
2407     my $nodehash = { map { $_ => 1 } @$nodelist };
2408     my $nodename = nodename();
2409
2410     PVE::QemuConfig->foreach_volume($conf, sub {
2411         my ($ds, $drive) = @_;
2412
2413         my $volid = $drive->{file};
2414         return if !$volid;
2415
2416         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2417         if ($storeid) {
2418             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2419             if ($scfg->{disable}) {
2420                 $nodehash = {};
2421             } elsif (my $avail = $scfg->{nodes}) {
2422                 foreach my $node (keys %$nodehash) {
2423                     delete $nodehash->{$node} if !$avail->{$node};
2424                 }
2425             } elsif (!$scfg->{shared}) {
2426                 foreach my $node (keys %$nodehash) {
2427                     delete $nodehash->{$node} if $node ne $nodename
2428                 }
2429             }
2430         }
2431     });
2432
2433     return $nodehash
2434 }
2435
2436 sub check_local_storage_availability {
2437     my ($conf, $storecfg) = @_;
2438
2439     my $nodelist = PVE::Cluster::get_nodelist();
2440     my $nodehash = { map { $_ => {} } @$nodelist };
2441
2442     PVE::QemuConfig->foreach_volume($conf, sub {
2443         my ($ds, $drive) = @_;
2444
2445         my $volid = $drive->{file};
2446         return if !$volid;
2447
2448         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2449         if ($storeid) {
2450             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2451
2452             if ($scfg->{disable}) {
2453                 foreach my $node (keys %$nodehash) {
2454                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2455                 }
2456             } elsif (my $avail = $scfg->{nodes}) {
2457                 foreach my $node (keys %$nodehash) {
2458                     if (!$avail->{$node}) {
2459                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2460                     }
2461                 }
2462             }
2463         }
2464     });
2465
2466     foreach my $node (values %$nodehash) {
2467         if (my $unavail = $node->{unavailable_storages}) {
2468             $node->{unavailable_storages} = [ sort keys %$unavail ];
2469         }
2470     }
2471
2472     return $nodehash
2473 }
2474
2475 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2476 sub check_running {
2477     my ($vmid, $nocheck, $node) = @_;
2478
2479     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2480     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2481 }
2482
2483 sub vzlist {
2484
2485     my $vzlist = config_list();
2486
2487     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2488
2489     while (defined(my $de = $fd->read)) {
2490         next if $de !~ m/^(\d+)\.pid$/;
2491         my $vmid = $1;
2492         next if !defined($vzlist->{$vmid});
2493         if (my $pid = check_running($vmid)) {
2494             $vzlist->{$vmid}->{pid} = $pid;
2495         }
2496     }
2497
2498     return $vzlist;
2499 }
2500
2501 our $vmstatus_return_properties = {
2502     vmid => get_standard_option('pve-vmid'),
2503     status => {
2504         description => "Qemu process status.",
2505         type => 'string',
2506         enum => ['stopped', 'running'],
2507     },
2508     maxmem => {
2509         description => "Maximum memory in bytes.",
2510         type => 'integer',
2511         optional => 1,
2512         renderer => 'bytes',
2513     },
2514     maxdisk => {
2515         description => "Root disk size in bytes.",
2516         type => 'integer',
2517         optional => 1,
2518         renderer => 'bytes',
2519     },
2520     name => {
2521         description => "VM name.",
2522         type => 'string',
2523         optional => 1,
2524     },
2525     qmpstatus => {
2526         description => "Qemu QMP agent status.",
2527         type => 'string',
2528         optional => 1,
2529     },
2530     pid => {
2531         description => "PID of running qemu process.",
2532         type => 'integer',
2533         optional => 1,
2534     },
2535     uptime => {
2536         description => "Uptime.",
2537         type => 'integer',
2538         optional => 1,
2539         renderer => 'duration',
2540     },
2541     cpus => {
2542         description => "Maximum usable CPUs.",
2543         type => 'number',
2544         optional => 1,
2545     },
2546     lock => {
2547         description => "The current config lock, if any.",
2548         type => 'string',
2549         optional => 1,
2550     },
2551     tags => {
2552         description  => "The current configured tags, if any",
2553         type => 'string',
2554         optional => 1,
2555     },
2556 };
2557
2558 my $last_proc_pid_stat;
2559
2560 # get VM status information
2561 # This must be fast and should not block ($full == false)
2562 # We only query KVM using QMP if $full == true (this can be slow)
2563 sub vmstatus {
2564     my ($opt_vmid, $full) = @_;
2565
2566     my $res = {};
2567
2568     my $storecfg = PVE::Storage::config();
2569
2570     my $list = vzlist();
2571     my $defaults = load_defaults();
2572
2573     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2574
2575     my $cpucount = $cpuinfo->{cpus} || 1;
2576
2577     foreach my $vmid (keys %$list) {
2578         next if $opt_vmid && ($vmid ne $opt_vmid);
2579
2580         my $conf = PVE::QemuConfig->load_config($vmid);
2581
2582         my $d = { vmid => $vmid };
2583         $d->{pid} = $list->{$vmid}->{pid};
2584
2585         # fixme: better status?
2586         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2587
2588         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2589         if (defined($size)) {
2590             $d->{disk} = 0; # no info available
2591             $d->{maxdisk} = $size;
2592         } else {
2593             $d->{disk} = 0;
2594             $d->{maxdisk} = 0;
2595         }
2596
2597         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2598             * ($conf->{cores} || $defaults->{cores});
2599         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2600         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2601
2602         $d->{name} = $conf->{name} || "VM $vmid";
2603         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2604             : $defaults->{memory}*(1024*1024);
2605
2606         if ($conf->{balloon}) {
2607             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2608             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2609                 : $defaults->{shares};
2610         }
2611
2612         $d->{uptime} = 0;
2613         $d->{cpu} = 0;
2614         $d->{mem} = 0;
2615
2616         $d->{netout} = 0;
2617         $d->{netin} = 0;
2618
2619         $d->{diskread} = 0;
2620         $d->{diskwrite} = 0;
2621
2622         $d->{template} = PVE::QemuConfig->is_template($conf);
2623
2624         $d->{serial} = 1 if conf_has_serial($conf);
2625         $d->{lock} = $conf->{lock} if $conf->{lock};
2626         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2627
2628         $res->{$vmid} = $d;
2629     }
2630
2631     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2632     foreach my $dev (keys %$netdev) {
2633         next if $dev !~ m/^tap([1-9]\d*)i/;
2634         my $vmid = $1;
2635         my $d = $res->{$vmid};
2636         next if !$d;
2637
2638         $d->{netout} += $netdev->{$dev}->{receive};
2639         $d->{netin} += $netdev->{$dev}->{transmit};
2640
2641         if ($full) {
2642             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2643             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2644         }
2645
2646     }
2647
2648     my $ctime = gettimeofday;
2649
2650     foreach my $vmid (keys %$list) {
2651
2652         my $d = $res->{$vmid};
2653         my $pid = $d->{pid};
2654         next if !$pid;
2655
2656         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2657         next if !$pstat; # not running
2658
2659         my $used = $pstat->{utime} + $pstat->{stime};
2660
2661         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2662
2663         if ($pstat->{vsize}) {
2664             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2665         }
2666
2667         my $old = $last_proc_pid_stat->{$pid};
2668         if (!$old) {
2669             $last_proc_pid_stat->{$pid} = {
2670                 time => $ctime,
2671                 used => $used,
2672                 cpu => 0,
2673             };
2674             next;
2675         }
2676
2677         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2678
2679         if ($dtime > 1000) {
2680             my $dutime = $used -  $old->{used};
2681
2682             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2683             $last_proc_pid_stat->{$pid} = {
2684                 time => $ctime,
2685                 used => $used,
2686                 cpu => $d->{cpu},
2687             };
2688         } else {
2689             $d->{cpu} = $old->{cpu};
2690         }
2691     }
2692
2693     return $res if !$full;
2694
2695     my $qmpclient = PVE::QMPClient->new();
2696
2697     my $ballooncb = sub {
2698         my ($vmid, $resp) = @_;
2699
2700         my $info = $resp->{'return'};
2701         return if !$info->{max_mem};
2702
2703         my $d = $res->{$vmid};
2704
2705         # use memory assigned to VM
2706         $d->{maxmem} = $info->{max_mem};
2707         $d->{balloon} = $info->{actual};
2708
2709         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2710             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2711             $d->{freemem} = $info->{free_mem};
2712         }
2713
2714         $d->{ballooninfo} = $info;
2715     };
2716
2717     my $blockstatscb = sub {
2718         my ($vmid, $resp) = @_;
2719         my $data = $resp->{'return'} || [];
2720         my $totalrdbytes = 0;
2721         my $totalwrbytes = 0;
2722
2723         for my $blockstat (@$data) {
2724             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2725             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2726
2727             $blockstat->{device} =~ s/drive-//;
2728             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2729         }
2730         $res->{$vmid}->{diskread} = $totalrdbytes;
2731         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2732     };
2733
2734     my $statuscb = sub {
2735         my ($vmid, $resp) = @_;
2736
2737         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2738         # this fails if ballon driver is not loaded, so this must be
2739         # the last commnand (following command are aborted if this fails).
2740         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2741
2742         my $status = 'unknown';
2743         if (!defined($status = $resp->{'return'}->{status})) {
2744             warn "unable to get VM status\n";
2745             return;
2746         }
2747
2748         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2749     };
2750
2751     foreach my $vmid (keys %$list) {
2752         next if $opt_vmid && ($vmid ne $opt_vmid);
2753         next if !$res->{$vmid}->{pid}; # not running
2754         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2755     }
2756
2757     $qmpclient->queue_execute(undef, 2);
2758
2759     foreach my $vmid (keys %$list) {
2760         next if $opt_vmid && ($vmid ne $opt_vmid);
2761         next if !$res->{$vmid}->{pid}; #not running
2762
2763         # we can't use the $qmpclient since it might have already aborted on
2764         # 'query-balloon', but this might also fail for older versions...
2765         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2766         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2767     }
2768
2769     foreach my $vmid (keys %$list) {
2770         next if $opt_vmid && ($vmid ne $opt_vmid);
2771         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2772     }
2773
2774     return $res;
2775 }
2776
2777 sub conf_has_serial {
2778     my ($conf) = @_;
2779
2780     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2781         if ($conf->{"serial$i"}) {
2782             return 1;
2783         }
2784     }
2785
2786     return 0;
2787 }
2788
2789 sub conf_has_audio {
2790     my ($conf, $id) = @_;
2791
2792     $id //= 0;
2793     my $audio = $conf->{"audio$id"};
2794     return if !defined($audio);
2795
2796     my $audioproperties = parse_property_string($audio_fmt, $audio);
2797     my $audiodriver = $audioproperties->{driver} // 'spice';
2798
2799     return {
2800         dev => $audioproperties->{device},
2801         dev_id => "audiodev$id",
2802         backend => $audiodriver,
2803         backend_id => "$audiodriver-backend${id}",
2804     };
2805 }
2806
2807 sub audio_devs {
2808     my ($audio, $audiopciaddr, $machine_version) = @_;
2809
2810     my $devs = [];
2811
2812     my $id = $audio->{dev_id};
2813     my $audiodev = "";
2814     if (min_version($machine_version, 4, 2)) {
2815         $audiodev = ",audiodev=$audio->{backend_id}";
2816     }
2817
2818     if ($audio->{dev} eq 'AC97') {
2819         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2820     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2821         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2822         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2823         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2824     } else {
2825         die "unkown audio device '$audio->{dev}', implement me!";
2826     }
2827
2828     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2829
2830     return $devs;
2831 }
2832
2833 sub vga_conf_has_spice {
2834     my ($vga) = @_;
2835
2836     my $vgaconf = parse_vga($vga);
2837     my $vgatype = $vgaconf->{type};
2838     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2839
2840     return $1 || 1;
2841 }
2842
2843 sub is_native($) {
2844     my ($arch) = @_;
2845     return get_host_arch() eq $arch;
2846 }
2847
2848 sub get_vm_arch {
2849     my ($conf) = @_;
2850     return $conf->{arch} // get_host_arch();
2851 }
2852
2853 my $default_machines = {
2854     x86_64 => 'pc',
2855     aarch64 => 'virt',
2856 };
2857
2858 sub get_vm_machine {
2859     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2860
2861     my $machine = $forcemachine || $conf->{machine};
2862
2863     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2864         $arch //= 'x86_64';
2865         $machine ||= $default_machines->{$arch};
2866         if ($add_pve_version) {
2867             $kvmversion //= kvm_user_version();
2868             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2869             $machine .= "+pve$pvever";
2870         }
2871     }
2872
2873     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2874         # for version-pinned machines that do not include a pve-version (e.g.
2875         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2876         $machine .= '+pve0';
2877     }
2878
2879     return $machine;
2880 }
2881
2882 sub get_ovmf_files($) {
2883     my ($arch) = @_;
2884
2885     my $ovmf = $OVMF->{$arch}
2886         or die "no OVMF images known for architecture '$arch'\n";
2887
2888     return @$ovmf;
2889 }
2890
2891 my $Arch2Qemu = {
2892     aarch64 => '/usr/bin/qemu-system-aarch64',
2893     x86_64 => '/usr/bin/qemu-system-x86_64',
2894 };
2895 sub get_command_for_arch($) {
2896     my ($arch) = @_;
2897     return '/usr/bin/kvm' if is_native($arch);
2898
2899     my $cmd = $Arch2Qemu->{$arch}
2900         or die "don't know how to emulate architecture '$arch'\n";
2901     return $cmd;
2902 }
2903
2904 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2905 # to use in a QEMU command line (-cpu element), first array_intersect the result
2906 # of query_supported_ with query_understood_. This is necessary because:
2907 #
2908 # a) query_understood_ returns flags the host cannot use and
2909 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2910 #    flags, but CPU settings - with most of them being flags. Those settings
2911 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2912 #
2913 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2914 # expensive. If you need the value returned from this, you can get it much
2915 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2916 # $accel being 'kvm' or 'tcg'.
2917 #
2918 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2919 # changes, automatically populating pmxcfs.
2920 #
2921 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2922 # since kvm and tcg machines support different flags
2923 #
2924 sub query_supported_cpu_flags {
2925     my ($arch) = @_;
2926
2927     $arch //= get_host_arch();
2928     my $default_machine = $default_machines->{$arch};
2929
2930     my $flags = {};
2931
2932     # FIXME: Once this is merged, the code below should work for ARM as well:
2933     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2934     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2935         $arch eq "aarch64";
2936
2937     my $kvm_supported = defined(kvm_version());
2938     my $qemu_cmd = get_command_for_arch($arch);
2939     my $fakevmid = -1;
2940     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2941
2942     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2943     my $query_supported_run_qemu = sub {
2944         my ($kvm) = @_;
2945
2946         my $flags = {};
2947         my $cmd = [
2948             $qemu_cmd,
2949             '-machine', $default_machine,
2950             '-display', 'none',
2951             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2952             '-mon', 'chardev=qmp,mode=control',
2953             '-pidfile', $pidfile,
2954             '-S', '-daemonize'
2955         ];
2956
2957         if (!$kvm) {
2958             push @$cmd, '-accel', 'tcg';
2959         }
2960
2961         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2962         die "QEMU flag querying VM exited with code " . $rc if $rc;
2963
2964         eval {
2965             my $cmd_result = mon_cmd(
2966                 $fakevmid,
2967                 'query-cpu-model-expansion',
2968                 type => 'full',
2969                 model => { name => 'host' }
2970             );
2971
2972             my $props = $cmd_result->{model}->{props};
2973             foreach my $prop (keys %$props) {
2974                 next if $props->{$prop} ne '1';
2975                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2976                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2977                 # We only keep those with underscores, to match /proc/cpuinfo
2978                 $prop =~ s/\.|-/_/g;
2979                 $flags->{$prop} = 1;
2980             }
2981         };
2982         my $err = $@;
2983
2984         # force stop with 10 sec timeout and 'nocheck'
2985         # always stop, even if QMP failed
2986         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2987
2988         die $err if $err;
2989
2990         return [ sort keys %$flags ];
2991     };
2992
2993     # We need to query QEMU twice, since KVM and TCG have different supported flags
2994     PVE::QemuConfig->lock_config($fakevmid, sub {
2995         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2996         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2997
2998         if ($kvm_supported) {
2999             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3000             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3001         }
3002     });
3003
3004     return $flags;
3005 }
3006
3007 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3008 my $understood_cpu_flag_dir = "/usr/share/kvm";
3009 sub query_understood_cpu_flags {
3010     my $arch = get_host_arch();
3011     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3012
3013     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3014         if ! -e $filepath;
3015
3016     my $raw = file_get_contents($filepath);
3017     $raw =~ s/^\s+|\s+$//g;
3018     my @flags = split(/\s+/, $raw);
3019
3020     return \@flags;
3021 }
3022
3023 sub config_to_command {
3024     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3025
3026     my $cmd = [];
3027     my $globalFlags = [];
3028     my $machineFlags = [];
3029     my $rtcFlags = [];
3030     my $devices = [];
3031     my $pciaddr = '';
3032     my $bridges = {};
3033     my $ostype = $conf->{ostype};
3034     my $winversion = windows_version($ostype);
3035     my $kvm = $conf->{kvm};
3036     my $nodename = nodename();
3037
3038     my $arch = get_vm_arch($conf);
3039     my $kvm_binary = get_command_for_arch($arch);
3040     my $kvmver = kvm_user_version($kvm_binary);
3041
3042     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3043         $kvmver //= "undefined";
3044         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3045     }
3046
3047     my $add_pve_version = min_version($kvmver, 4, 1);
3048
3049     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3050     my $machine_version = extract_version($machine_type, $kvmver);
3051     $kvm //= 1 if is_native($arch);
3052
3053     $machine_version =~ m/(\d+)\.(\d+)/;
3054     my ($machine_major, $machine_minor) = ($1, $2);
3055
3056     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3057         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3058     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3059         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3060             ." please upgrade node '$nodename'\n"
3061     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3062         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3063         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3064             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3065             ." node '$nodename'\n";
3066     }
3067
3068     # if a specific +pve version is required for a feature, use $version_guard
3069     # instead of min_version to allow machines to be run with the minimum
3070     # required version
3071     my $required_pve_version = 0;
3072     my $version_guard = sub {
3073         my ($major, $minor, $pve) = @_;
3074         return 0 if !min_version($machine_version, $major, $minor, $pve);
3075         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3076         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3077         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3078         return 1;
3079     };
3080
3081     if ($kvm && !defined kvm_version()) {
3082         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3083             ." or enable in BIOS.\n";
3084     }
3085
3086     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3087     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3088     my $use_old_bios_files = undef;
3089     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3090
3091     my $cpuunits = defined($conf->{cpuunits}) ?
3092             $conf->{cpuunits} : $defaults->{cpuunits};
3093
3094     push @$cmd, $kvm_binary;
3095
3096     push @$cmd, '-id', $vmid;
3097
3098     my $vmname = $conf->{name} || "vm$vmid";
3099
3100     push @$cmd, '-name', $vmname;
3101
3102     push @$cmd, '-no-shutdown';
3103
3104     my $use_virtio = 0;
3105
3106     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3107     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3108     push @$cmd, '-mon', "chardev=qmp,mode=control";
3109
3110     if (min_version($machine_version, 2, 12)) {
3111         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3112         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3113     }
3114
3115     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3116
3117     push @$cmd, '-daemonize';
3118
3119     if ($conf->{smbios1}) {
3120         my $smbios_conf = parse_smbios1($conf->{smbios1});
3121         if ($smbios_conf->{base64}) {
3122             # Do not pass base64 flag to qemu
3123             delete $smbios_conf->{base64};
3124             my $smbios_string = "";
3125             foreach my $key (keys %$smbios_conf) {
3126                 my $value;
3127                 if ($key eq "uuid") {
3128                     $value = $smbios_conf->{uuid}
3129                 } else {
3130                     $value = decode_base64($smbios_conf->{$key});
3131                 }
3132                 # qemu accepts any binary data, only commas need escaping by double comma
3133                 $value =~ s/,/,,/g;
3134                 $smbios_string .= "," . $key . "=" . $value if $value;
3135             }
3136             push @$cmd, '-smbios', "type=1" . $smbios_string;
3137         } else {
3138             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3139         }
3140     }
3141
3142     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3143         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3144         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3145
3146         my ($path, $format);
3147         if (my $efidisk = $conf->{efidisk0}) {
3148             my $d = parse_drive('efidisk0', $efidisk);
3149             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3150             $format = $d->{format};
3151             if ($storeid) {
3152                 $path = PVE::Storage::path($storecfg, $d->{file});
3153                 if (!defined($format)) {
3154                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3155                     $format = qemu_img_format($scfg, $volname);
3156                 }
3157             } else {
3158                 $path = $d->{file};
3159                 die "efidisk format must be specified\n"
3160                     if !defined($format);
3161             }
3162         } else {
3163             warn "no efidisk configured! Using temporary efivars disk.\n";
3164             $path = "/tmp/$vmid-ovmf.fd";
3165             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3166             $format = 'raw';
3167         }
3168
3169         my $size_str = "";
3170
3171         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3172             $size_str = ",size=" . (-s $ovmf_vars);
3173         }
3174
3175         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3176         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3177     }
3178
3179     # load q35 config
3180     if ($q35) {
3181         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3182         if (min_version($machine_version, 4, 0)) {
3183             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3184         } else {
3185             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3186         }
3187     }
3188
3189     if ($conf->{vmgenid}) {
3190         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3191     }
3192
3193     # add usb controllers
3194     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3195         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3196     push @$devices, @usbcontrollers if @usbcontrollers;
3197     my $vga = parse_vga($conf->{vga});
3198
3199     my $qxlnum = vga_conf_has_spice($conf->{vga});
3200     $vga->{type} = 'qxl' if $qxlnum;
3201
3202     if (!$vga->{type}) {
3203         if ($arch eq 'aarch64') {
3204             $vga->{type} = 'virtio';
3205         } elsif (min_version($machine_version, 2, 9)) {
3206             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3207         } else {
3208             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3209         }
3210     }
3211
3212     # enable absolute mouse coordinates (needed by vnc)
3213     my $tablet;
3214     if (defined($conf->{tablet})) {
3215         $tablet = $conf->{tablet};
3216     } else {
3217         $tablet = $defaults->{tablet};
3218         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3219         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3220     }
3221
3222     if ($tablet) {
3223         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3224         my $kbd = print_keyboarddevice_full($conf, $arch);
3225         push @$devices, '-device', $kbd if defined($kbd);
3226     }
3227
3228     my $bootorder = device_bootorder($conf);
3229
3230     # host pci device passthrough
3231     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3232         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3233
3234     # usb devices
3235     my $usb_dev_features = {};
3236     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3237
3238     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3239         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3240     push @$devices, @usbdevices if @usbdevices;
3241
3242     # serial devices
3243     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3244         if (my $path = $conf->{"serial$i"}) {
3245             if ($path eq 'socket') {
3246                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3247                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3248                 # On aarch64, serial0 is the UART device. Qemu only allows
3249                 # connecting UART devices via the '-serial' command line, as
3250                 # the device has a fixed slot on the hardware...
3251                 if ($arch eq 'aarch64' && $i == 0) {
3252                     push @$devices, '-serial', "chardev:serial$i";
3253                 } else {
3254                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3255                 }
3256             } else {
3257                 die "no such serial device\n" if ! -c $path;
3258                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3259                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3260             }
3261         }
3262     }
3263
3264     # parallel devices
3265     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3266         if (my $path = $conf->{"parallel$i"}) {
3267             die "no such parallel device\n" if ! -c $path;
3268             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3269             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3270             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3271         }
3272     }
3273
3274     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3275         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3276         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3277         push @$devices, @$audio_devs;
3278     }
3279
3280     my $sockets = 1;
3281     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3282     $sockets = $conf->{sockets} if  $conf->{sockets};
3283
3284     my $cores = $conf->{cores} || 1;
3285
3286     my $maxcpus = $sockets * $cores;
3287
3288     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3289
3290     my $allowed_vcpus = $cpuinfo->{cpus};
3291
3292     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3293         if ($allowed_vcpus < $maxcpus);
3294
3295     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3296
3297         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3298         for (my $i = 2; $i <= $vcpus; $i++)  {
3299             my $cpustr = print_cpu_device($conf,$i);
3300             push @$cmd, '-device', $cpustr;
3301         }
3302
3303     } else {
3304
3305         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3306     }
3307     push @$cmd, '-nodefaults';
3308
3309     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3310
3311     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3312
3313     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3314
3315     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3316         push @$devices, '-device', print_vga_device(
3317             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3318         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3319         push @$cmd,  '-vnc', "unix:$socket,password";
3320     } else {
3321         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3322         push @$cmd, '-nographic';
3323     }
3324
3325     # time drift fix
3326     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3327     my $useLocaltime = $conf->{localtime};
3328
3329     if ($winversion >= 5) { # windows
3330         $useLocaltime = 1 if !defined($conf->{localtime});
3331
3332         # use time drift fix when acpi is enabled
3333         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3334             $tdf = 1 if !defined($conf->{tdf});
3335         }
3336     }
3337
3338     if ($winversion >= 6) {
3339         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3340         push @$cmd, '-no-hpet';
3341     }
3342
3343     push @$rtcFlags, 'driftfix=slew' if $tdf;
3344
3345     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3346         push @$rtcFlags, "base=$conf->{startdate}";
3347     } elsif ($useLocaltime) {
3348         push @$rtcFlags, 'base=localtime';
3349     }
3350
3351     if ($forcecpu) {
3352         push @$cmd, '-cpu', $forcecpu;
3353     } else {
3354         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3355     }
3356
3357     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3358
3359     push @$cmd, '-S' if $conf->{freeze};
3360
3361     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3362
3363     my $guest_agent = parse_guest_agent($conf);
3364
3365     if ($guest_agent->{enabled}) {
3366         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3367         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3368
3369         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3370             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3371             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3372             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3373         } elsif ($guest_agent->{type} eq 'isa') {
3374             push @$devices, '-device', "isa-serial,chardev=qga0";
3375         }
3376     }
3377
3378     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3379     if ($rng && $version_guard->(4, 1, 2)) {
3380         check_rng_source($rng->{source});
3381
3382         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3383         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3384         my $limiter_str = "";
3385         if ($max_bytes) {
3386             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3387         }
3388
3389         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3390         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3391         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3392     }
3393
3394     my $spice_port;
3395
3396     if ($qxlnum) {
3397         if ($qxlnum > 1) {
3398             if ($winversion){
3399                 for (my $i = 1; $i < $qxlnum; $i++){
3400                     push @$devices, '-device', print_vga_device(
3401                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3402                 }
3403             } else {
3404                 # assume other OS works like Linux
3405                 my ($ram, $vram) = ("134217728", "67108864");
3406                 if ($vga->{memory}) {
3407                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3408                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3409                 }
3410                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3411                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3412             }
3413         }
3414
3415         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3416
3417         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3418         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3419         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3420
3421         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3422         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3423         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3424
3425         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3426         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3427
3428         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3429         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3430         if ($spice_enhancement->{foldersharing}) {
3431             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3432             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3433         }
3434
3435         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3436         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3437             if $spice_enhancement->{videostreaming};
3438
3439         push @$devices, '-spice', "$spice_opts";
3440     }
3441
3442     # enable balloon by default, unless explicitly disabled
3443     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3444         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3445         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3446     }
3447
3448     if ($conf->{watchdog}) {
3449         my $wdopts = parse_watchdog($conf->{watchdog});
3450         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3451         my $watchdog = $wdopts->{model} || 'i6300esb';
3452         push @$devices, '-device', "$watchdog$pciaddr";
3453         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3454     }
3455
3456     my $vollist = [];
3457     my $scsicontroller = {};
3458     my $ahcicontroller = {};
3459     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3460
3461     # Add iscsi initiator name if available
3462     if (my $initiator = get_initiator_name()) {
3463         push @$devices, '-iscsi', "initiator-name=$initiator";
3464     }
3465
3466     PVE::QemuConfig->foreach_volume($conf, sub {
3467         my ($ds, $drive) = @_;
3468
3469         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3470             push @$vollist, $drive->{file};
3471         }
3472
3473         # ignore efidisk here, already added in bios/fw handling code above
3474         return if $drive->{interface} eq 'efidisk';
3475
3476         $use_virtio = 1 if $ds =~ m/^virtio/;
3477
3478         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3479
3480         if ($drive->{interface} eq 'virtio'){
3481            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3482         }
3483
3484         if ($drive->{interface} eq 'scsi') {
3485
3486             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3487
3488             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3489                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3490
3491             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3492             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3493
3494             my $iothread = '';
3495             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3496                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3497                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3498             } elsif ($drive->{iothread}) {
3499                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3500             }
3501
3502             my $queues = '';
3503             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3504                 $queues = ",num_queues=$drive->{queues}";
3505             }
3506
3507             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3508                 if !$scsicontroller->{$controller};
3509             $scsicontroller->{$controller}=1;
3510         }
3511
3512         if ($drive->{interface} eq 'sata') {
3513             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3514             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3515             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3516                 if !$ahcicontroller->{$controller};
3517             $ahcicontroller->{$controller}=1;
3518         }
3519
3520         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3521         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3522
3523         push @$devices, '-drive',$drive_cmd;
3524         push @$devices, '-device', print_drivedevice_full(
3525             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3526     });
3527
3528     for (my $i = 0; $i < $MAX_NETS; $i++) {
3529         my $netname = "net$i";
3530
3531         next if !$conf->{$netname};
3532         my $d = parse_net($conf->{$netname});
3533         next if !$d;
3534
3535         $use_virtio = 1 if $d->{model} eq 'virtio';
3536
3537         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3538
3539         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3540         push @$devices, '-netdev', $netdevfull;
3541
3542         my $netdevicefull = print_netdevice_full(
3543             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3544
3545         push @$devices, '-device', $netdevicefull;
3546     }
3547
3548     if ($conf->{ivshmem}) {
3549         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3550
3551         my $bus;
3552         if ($q35) {
3553             $bus = print_pcie_addr("ivshmem");
3554         } else {
3555             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3556         }
3557
3558         my $ivshmem_name = $ivshmem->{name} // $vmid;
3559         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3560
3561         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3562         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3563             .",size=$ivshmem->{size}M";
3564     }
3565
3566     # pci.4 is nested in pci.1
3567     $bridges->{1} = 1 if $bridges->{4};
3568
3569     if (!$q35) {
3570         # add pci bridges
3571         if (min_version($machine_version, 2, 3)) {
3572            $bridges->{1} = 1;
3573            $bridges->{2} = 1;
3574         }
3575
3576         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3577
3578     }
3579
3580     for my $k (sort {$b cmp $a} keys %$bridges) {
3581         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3582
3583         my $k_name = $k;
3584         if ($k == 2 && $legacy_igd) {
3585             $k_name = "$k-igd";
3586         }
3587         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3588
3589         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3590         if ($q35) {
3591             # add after -readconfig pve-q35.cfg
3592             splice @$devices, 2, 0, '-device', $devstr;
3593         } else {
3594             unshift @$devices, '-device', $devstr if $k > 0;
3595         }
3596     }
3597
3598     if (!$kvm) {
3599         push @$machineFlags, 'accel=tcg';
3600     }
3601
3602     my $machine_type_min = $machine_type;
3603     if ($add_pve_version) {
3604         $machine_type_min =~ s/\+pve\d+$//;
3605         $machine_type_min .= "+pve$required_pve_version";
3606     }
3607     push @$machineFlags, "type=${machine_type_min}";
3608
3609     push @$cmd, @$devices;
3610     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3611     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3612     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3613
3614     if (my $vmstate = $conf->{vmstate}) {
3615         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3616         push @$vollist, $vmstate;
3617         push @$cmd, '-loadstate', $statepath;
3618         print "activating and using '$vmstate' as vmstate\n";
3619     }
3620
3621     # add custom args
3622     if ($conf->{args}) {
3623         my $aa = PVE::Tools::split_args($conf->{args});
3624         push @$cmd, @$aa;
3625     }
3626
3627     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3628 }
3629
3630 sub check_rng_source {
3631     my ($source) = @_;
3632
3633     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3634     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3635         if ! -e $source;
3636
3637     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3638     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3639         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3640         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3641         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3642             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3643             ." to the host.\n";
3644     }
3645 }
3646
3647 sub spice_port {
3648     my ($vmid) = @_;
3649
3650     my $res = mon_cmd($vmid, 'query-spice');
3651
3652     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3653 }
3654
3655 sub vm_devices_list {
3656     my ($vmid) = @_;
3657
3658     my $res = mon_cmd($vmid, 'query-pci');
3659     my $devices_to_check = [];
3660     my $devices = {};
3661     foreach my $pcibus (@$res) {
3662         push @$devices_to_check, @{$pcibus->{devices}},
3663     }
3664
3665     while (@$devices_to_check) {
3666         my $to_check = [];
3667         for my $d (@$devices_to_check) {
3668             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3669             next if !$d->{'pci_bridge'};
3670
3671             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3672             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3673         }
3674         $devices_to_check = $to_check;
3675     }
3676
3677     my $resblock = mon_cmd($vmid, 'query-block');
3678     foreach my $block (@$resblock) {
3679         if($block->{device} =~ m/^drive-(\S+)/){
3680                 $devices->{$1} = 1;
3681         }
3682     }
3683
3684     my $resmice = mon_cmd($vmid, 'query-mice');
3685     foreach my $mice (@$resmice) {
3686         if ($mice->{name} eq 'QEMU HID Tablet') {
3687             $devices->{tablet} = 1;
3688             last;
3689         }
3690     }
3691
3692     # for usb devices there is no query-usb
3693     # but we can iterate over the entries in
3694     # qom-list path=/machine/peripheral
3695     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3696     foreach my $per (@$resperipheral) {
3697         if ($per->{name} =~ m/^usb\d+$/) {
3698             $devices->{$per->{name}} = 1;
3699         }
3700     }
3701
3702     return $devices;
3703 }
3704
3705 sub vm_deviceplug {
3706     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3707
3708     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3709
3710     my $devices_list = vm_devices_list($vmid);
3711     return 1 if defined($devices_list->{$deviceid});
3712
3713     # add PCI bridge if we need it for the device
3714     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3715
3716     if ($deviceid eq 'tablet') {
3717
3718         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3719
3720     } elsif ($deviceid eq 'keyboard') {
3721
3722         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3723
3724     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3725
3726         die "usb hotplug currently not reliable\n";
3727         # since we can't reliably hot unplug all added usb devices and usb
3728         # passthrough breaks live migration we disable usb hotplugging for now
3729         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3730
3731     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3732
3733         qemu_iothread_add($vmid, $deviceid, $device);
3734
3735         qemu_driveadd($storecfg, $vmid, $device);
3736         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3737
3738         qemu_deviceadd($vmid, $devicefull);
3739         eval { qemu_deviceaddverify($vmid, $deviceid); };
3740         if (my $err = $@) {
3741             eval { qemu_drivedel($vmid, $deviceid); };
3742             warn $@ if $@;
3743             die $err;
3744         }
3745
3746     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3747
3748
3749         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3750         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3751         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3752
3753         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3754
3755         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3756             qemu_iothread_add($vmid, $deviceid, $device);
3757             $devicefull .= ",iothread=iothread-$deviceid";
3758         }
3759
3760         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3761             $devicefull .= ",num_queues=$device->{queues}";
3762         }
3763
3764         qemu_deviceadd($vmid, $devicefull);
3765         qemu_deviceaddverify($vmid, $deviceid);
3766
3767     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3768
3769         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3770         qemu_driveadd($storecfg, $vmid, $device);
3771
3772         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3773         eval { qemu_deviceadd($vmid, $devicefull); };
3774         if (my $err = $@) {
3775             eval { qemu_drivedel($vmid, $deviceid); };
3776             warn $@ if $@;
3777             die $err;
3778         }
3779
3780     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3781
3782         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3783
3784         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3785         my $use_old_bios_files = undef;
3786         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3787
3788         my $netdevicefull = print_netdevice_full(
3789             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3790         qemu_deviceadd($vmid, $netdevicefull);
3791         eval {
3792             qemu_deviceaddverify($vmid, $deviceid);
3793             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3794         };
3795         if (my $err = $@) {
3796             eval { qemu_netdevdel($vmid, $deviceid); };
3797             warn $@ if $@;
3798             die $err;
3799         }
3800
3801     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3802
3803         my $bridgeid = $2;
3804         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3805         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3806
3807         qemu_deviceadd($vmid, $devicefull);
3808         qemu_deviceaddverify($vmid, $deviceid);
3809
3810     } else {
3811         die "can't hotplug device '$deviceid'\n";
3812     }
3813
3814     return 1;
3815 }
3816
3817 # fixme: this should raise exceptions on error!
3818 sub vm_deviceunplug {
3819     my ($vmid, $conf, $deviceid) = @_;
3820
3821     my $devices_list = vm_devices_list($vmid);
3822     return 1 if !defined($devices_list->{$deviceid});
3823
3824     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3825     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3826
3827     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3828
3829         qemu_devicedel($vmid, $deviceid);
3830
3831     } elsif ($deviceid =~ m/^usb\d+$/) {
3832
3833         die "usb hotplug currently not reliable\n";
3834         # when unplugging usb devices this way, there may be remaining usb
3835         # controllers/hubs so we disable it for now
3836         #qemu_devicedel($vmid, $deviceid);
3837         #qemu_devicedelverify($vmid, $deviceid);
3838
3839     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3840
3841         qemu_devicedel($vmid, $deviceid);
3842         qemu_devicedelverify($vmid, $deviceid);
3843         qemu_drivedel($vmid, $deviceid);
3844         qemu_iothread_del($conf, $vmid, $deviceid);
3845
3846     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3847
3848         qemu_devicedel($vmid, $deviceid);
3849         qemu_devicedelverify($vmid, $deviceid);
3850         qemu_iothread_del($conf, $vmid, $deviceid);
3851
3852     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3853
3854         qemu_devicedel($vmid, $deviceid);
3855         qemu_drivedel($vmid, $deviceid);
3856         qemu_deletescsihw($conf, $vmid, $deviceid);
3857
3858     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3859
3860         qemu_devicedel($vmid, $deviceid);
3861         qemu_devicedelverify($vmid, $deviceid);
3862         qemu_netdevdel($vmid, $deviceid);
3863
3864     } else {
3865         die "can't unplug device '$deviceid'\n";
3866     }
3867
3868     return 1;
3869 }
3870
3871 sub qemu_deviceadd {
3872     my ($vmid, $devicefull) = @_;
3873
3874     $devicefull = "driver=".$devicefull;
3875     my %options =  split(/[=,]/, $devicefull);
3876
3877     mon_cmd($vmid, "device_add" , %options);
3878 }
3879
3880 sub qemu_devicedel {
3881     my ($vmid, $deviceid) = @_;
3882
3883     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3884 }
3885
3886 sub qemu_iothread_add {
3887     my($vmid, $deviceid, $device) = @_;
3888
3889     if ($device->{iothread}) {
3890         my $iothreads = vm_iothreads_list($vmid);
3891         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3892     }
3893 }
3894
3895 sub qemu_iothread_del {
3896     my($conf, $vmid, $deviceid) = @_;
3897
3898     my $confid = $deviceid;
3899     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3900         $confid = 'scsi' . $1;
3901     }
3902     my $device = parse_drive($confid, $conf->{$confid});
3903     if ($device->{iothread}) {
3904         my $iothreads = vm_iothreads_list($vmid);
3905         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3906     }
3907 }
3908
3909 sub qemu_objectadd {
3910     my($vmid, $objectid, $qomtype) = @_;
3911
3912     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3913
3914     return 1;
3915 }
3916
3917 sub qemu_objectdel {
3918     my($vmid, $objectid) = @_;
3919
3920     mon_cmd($vmid, "object-del", id => $objectid);
3921
3922     return 1;
3923 }
3924
3925 sub qemu_driveadd {
3926     my ($storecfg, $vmid, $device) = @_;
3927
3928     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3929     $drive =~ s/\\/\\\\/g;
3930     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3931
3932     # If the command succeeds qemu prints: "OK"
3933     return 1 if $ret =~ m/OK/s;
3934
3935     die "adding drive failed: $ret\n";
3936 }
3937
3938 sub qemu_drivedel {
3939     my($vmid, $deviceid) = @_;
3940
3941     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3942     $ret =~ s/^\s+//;
3943
3944     return 1 if $ret eq "";
3945
3946     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3947     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3948
3949     die "deleting drive $deviceid failed : $ret\n";
3950 }
3951
3952 sub qemu_deviceaddverify {
3953     my ($vmid, $deviceid) = @_;
3954
3955     for (my $i = 0; $i <= 5; $i++) {
3956          my $devices_list = vm_devices_list($vmid);
3957          return 1 if defined($devices_list->{$deviceid});
3958          sleep 1;
3959     }
3960
3961     die "error on hotplug device '$deviceid'\n";
3962 }
3963
3964
3965 sub qemu_devicedelverify {
3966     my ($vmid, $deviceid) = @_;
3967
3968     # need to verify that the device is correctly removed as device_del
3969     # is async and empty return is not reliable
3970
3971     for (my $i = 0; $i <= 5; $i++) {
3972          my $devices_list = vm_devices_list($vmid);
3973          return 1 if !defined($devices_list->{$deviceid});
3974          sleep 1;
3975     }
3976
3977     die "error on hot-unplugging device '$deviceid'\n";
3978 }
3979
3980 sub qemu_findorcreatescsihw {
3981     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3982
3983     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3984
3985     my $scsihwid="$controller_prefix$controller";
3986     my $devices_list = vm_devices_list($vmid);
3987
3988     if(!defined($devices_list->{$scsihwid})) {
3989         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3990     }
3991
3992     return 1;
3993 }
3994
3995 sub qemu_deletescsihw {
3996     my ($conf, $vmid, $opt) = @_;
3997
3998     my $device = parse_drive($opt, $conf->{$opt});
3999
4000     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4001         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4002         return 1;
4003     }
4004
4005     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4006
4007     my $devices_list = vm_devices_list($vmid);
4008     foreach my $opt (keys %{$devices_list}) {
4009         if (is_valid_drivename($opt)) {
4010             my $drive = parse_drive($opt, $conf->{$opt});
4011             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4012                 return 1;
4013             }
4014         }
4015     }
4016
4017     my $scsihwid="scsihw$controller";
4018
4019     vm_deviceunplug($vmid, $conf, $scsihwid);
4020
4021     return 1;
4022 }
4023
4024 sub qemu_add_pci_bridge {
4025     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4026
4027     my $bridges = {};
4028
4029     my $bridgeid;
4030
4031     print_pci_addr($device, $bridges, $arch, $machine_type);
4032
4033     while (my ($k, $v) = each %$bridges) {
4034         $bridgeid = $k;
4035     }
4036     return 1 if !defined($bridgeid) || $bridgeid < 1;
4037
4038     my $bridge = "pci.$bridgeid";
4039     my $devices_list = vm_devices_list($vmid);
4040
4041     if (!defined($devices_list->{$bridge})) {
4042         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4043     }
4044
4045     return 1;
4046 }
4047
4048 sub qemu_set_link_status {
4049     my ($vmid, $device, $up) = @_;
4050
4051     mon_cmd($vmid, "set_link", name => $device,
4052                up => $up ? JSON::true : JSON::false);
4053 }
4054
4055 sub qemu_netdevadd {
4056     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4057
4058     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4059     my %options =  split(/[=,]/, $netdev);
4060
4061     if (defined(my $vhost = $options{vhost})) {
4062         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4063     }
4064
4065     if (defined(my $queues = $options{queues})) {
4066         $options{queues} = $queues + 0;
4067     }
4068
4069     mon_cmd($vmid, "netdev_add",  %options);
4070     return 1;
4071 }
4072
4073 sub qemu_netdevdel {
4074     my ($vmid, $deviceid) = @_;
4075
4076     mon_cmd($vmid, "netdev_del", id => $deviceid);
4077 }
4078
4079 sub qemu_usb_hotplug {
4080     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4081
4082     return if !$device;
4083
4084     # remove the old one first
4085     vm_deviceunplug($vmid, $conf, $deviceid);
4086
4087     # check if xhci controller is necessary and available
4088     if ($device->{usb3}) {
4089
4090         my $devicelist = vm_devices_list($vmid);
4091
4092         if (!$devicelist->{xhci}) {
4093             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4094             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4095         }
4096     }
4097     my $d = parse_usb_device($device->{host});
4098     $d->{usb3} = $device->{usb3};
4099
4100     # add the new one
4101     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4102 }
4103
4104 sub qemu_cpu_hotplug {
4105     my ($vmid, $conf, $vcpus) = @_;
4106
4107     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4108
4109     my $sockets = 1;
4110     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4111     $sockets = $conf->{sockets} if  $conf->{sockets};
4112     my $cores = $conf->{cores} || 1;
4113     my $maxcpus = $sockets * $cores;
4114
4115     $vcpus = $maxcpus if !$vcpus;
4116
4117     die "you can't add more vcpus than maxcpus\n"
4118         if $vcpus > $maxcpus;
4119
4120     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4121
4122     if ($vcpus < $currentvcpus) {
4123
4124         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4125
4126             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4127                 qemu_devicedel($vmid, "cpu$i");
4128                 my $retry = 0;
4129                 my $currentrunningvcpus = undef;
4130                 while (1) {
4131                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4132                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4133                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4134                     $retry++;
4135                     sleep 1;
4136                 }
4137                 #update conf after each succesfull cpu unplug
4138                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4139                 PVE::QemuConfig->write_config($vmid, $conf);
4140             }
4141         } else {
4142             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4143         }
4144
4145         return;
4146     }
4147
4148     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4149     die "vcpus in running vm does not match its configuration\n"
4150         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4151
4152     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4153
4154         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4155             my $cpustr = print_cpu_device($conf, $i);
4156             qemu_deviceadd($vmid, $cpustr);
4157
4158             my $retry = 0;
4159             my $currentrunningvcpus = undef;
4160             while (1) {
4161                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4162                 last if scalar(@{$currentrunningvcpus}) == $i;
4163                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4164                 sleep 1;
4165                 $retry++;
4166             }
4167             #update conf after each succesfull cpu hotplug
4168             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4169             PVE::QemuConfig->write_config($vmid, $conf);
4170         }
4171     } else {
4172
4173         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4174             mon_cmd($vmid, "cpu-add", id => int($i));
4175         }
4176     }
4177 }
4178
4179 sub qemu_block_set_io_throttle {
4180     my ($vmid, $deviceid,
4181         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4182         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4183         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4184         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4185
4186     return if !check_running($vmid) ;
4187
4188     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4189         bps => int($bps),
4190         bps_rd => int($bps_rd),
4191         bps_wr => int($bps_wr),
4192         iops => int($iops),
4193         iops_rd => int($iops_rd),
4194         iops_wr => int($iops_wr),
4195         bps_max => int($bps_max),
4196         bps_rd_max => int($bps_rd_max),
4197         bps_wr_max => int($bps_wr_max),
4198         iops_max => int($iops_max),
4199         iops_rd_max => int($iops_rd_max),
4200         iops_wr_max => int($iops_wr_max),
4201         bps_max_length => int($bps_max_length),
4202         bps_rd_max_length => int($bps_rd_max_length),
4203         bps_wr_max_length => int($bps_wr_max_length),
4204         iops_max_length => int($iops_max_length),
4205         iops_rd_max_length => int($iops_rd_max_length),
4206         iops_wr_max_length => int($iops_wr_max_length),
4207     );
4208
4209 }
4210
4211 sub qemu_block_resize {
4212     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4213
4214     my $running = check_running($vmid);
4215
4216     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4217
4218     return if !$running;
4219
4220     my $padding = (1024 - $size % 1024) % 1024;
4221     $size = $size + $padding;
4222
4223     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4224
4225 }
4226
4227 sub qemu_volume_snapshot {
4228     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4229
4230     my $running = check_running($vmid);
4231
4232     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4233         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4234     } else {
4235         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4236     }
4237 }
4238
4239 sub qemu_volume_snapshot_delete {
4240     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4241
4242     my $running = check_running($vmid);
4243
4244     if($running) {
4245
4246         $running = undef;
4247         my $conf = PVE::QemuConfig->load_config($vmid);
4248         PVE::QemuConfig->foreach_volume($conf, sub {
4249             my ($ds, $drive) = @_;
4250             $running = 1 if $drive->{file} eq $volid;
4251         });
4252     }
4253
4254     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4255         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4256     } else {
4257         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4258     }
4259 }
4260
4261 sub set_migration_caps {
4262     my ($vmid) = @_;
4263
4264     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4265
4266     my $cap_ref = [];
4267
4268     my $enabled_cap = {
4269         "auto-converge" => 1,
4270         "xbzrle" => 1,
4271         "x-rdma-pin-all" => 0,
4272         "zero-blocks" => 0,
4273         "compress" => 0,
4274         "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0,
4275     };
4276
4277     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4278
4279     for my $supported_capability (@$supported_capabilities) {
4280         push @$cap_ref, {
4281             capability => $supported_capability->{capability},
4282             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4283         };
4284     }
4285
4286     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4287 }
4288
4289 sub foreach_volid {
4290     my ($conf, $func, @param) = @_;
4291
4292     my $volhash = {};
4293
4294     my $test_volid = sub {
4295         my ($key, $drive, $snapname) = @_;
4296
4297         my $volid = $drive->{file};
4298         return if !$volid;
4299
4300         $volhash->{$volid}->{cdrom} //= 1;
4301         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4302
4303         my $replicate = $drive->{replicate} // 1;
4304         $volhash->{$volid}->{replicate} //= 0;
4305         $volhash->{$volid}->{replicate} = 1 if $replicate;
4306
4307         $volhash->{$volid}->{shared} //= 0;
4308         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4309
4310         $volhash->{$volid}->{referenced_in_config} //= 0;
4311         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4312
4313         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4314             if defined($snapname);
4315
4316         my $size = $drive->{size};
4317         $volhash->{$volid}->{size} //= $size if $size;
4318
4319         $volhash->{$volid}->{is_vmstate} //= 0;
4320         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4321
4322         $volhash->{$volid}->{is_unused} //= 0;
4323         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4324     };
4325
4326     my $include_opts = {
4327         extra_keys => ['vmstate'],
4328         include_unused => 1,
4329     };
4330
4331     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4332     foreach my $snapname (keys %{$conf->{snapshots}}) {
4333         my $snap = $conf->{snapshots}->{$snapname};
4334         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4335     }
4336
4337     foreach my $volid (keys %$volhash) {
4338         &$func($volid, $volhash->{$volid}, @param);
4339     }
4340 }
4341
4342 my $fast_plug_option = {
4343     'lock' => 1,
4344     'name' => 1,
4345     'onboot' => 1,
4346     'shares' => 1,
4347     'startup' => 1,
4348     'description' => 1,
4349     'protection' => 1,
4350     'vmstatestorage' => 1,
4351     'hookscript' => 1,
4352     'tags' => 1,
4353 };
4354
4355 # hotplug changes in [PENDING]
4356 # $selection hash can be used to only apply specified options, for
4357 # example: { cores => 1 } (only apply changed 'cores')
4358 # $errors ref is used to return error messages
4359 sub vmconfig_hotplug_pending {
4360     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4361
4362     my $defaults = load_defaults();
4363     my $arch = get_vm_arch($conf);
4364     my $machine_type = get_vm_machine($conf, undef, $arch);
4365
4366     # commit values which do not have any impact on running VM first
4367     # Note: those option cannot raise errors, we we do not care about
4368     # $selection and always apply them.
4369
4370     my $add_error = sub {
4371         my ($opt, $msg) = @_;
4372         $errors->{$opt} = "hotplug problem - $msg";
4373     };
4374
4375     my $changes = 0;
4376     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4377         if ($fast_plug_option->{$opt}) {
4378             $conf->{$opt} = $conf->{pending}->{$opt};
4379             delete $conf->{pending}->{$opt};
4380             $changes = 1;
4381         }
4382     }
4383
4384     if ($changes) {
4385         PVE::QemuConfig->write_config($vmid, $conf);
4386     }
4387
4388     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4389
4390     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4391     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4392     foreach my $opt (sort keys %$pending_delete_hash) {
4393         next if $selection && !$selection->{$opt};
4394         my $force = $pending_delete_hash->{$opt}->{force};
4395         eval {
4396             if ($opt eq 'hotplug') {
4397                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4398             } elsif ($opt eq 'tablet') {
4399                 die "skip\n" if !$hotplug_features->{usb};
4400                 if ($defaults->{tablet}) {
4401                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4402                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4403                         if $arch eq 'aarch64';
4404                 } else {
4405                     vm_deviceunplug($vmid, $conf, 'tablet');
4406                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4407                 }
4408             } elsif ($opt =~ m/^usb\d+/) {
4409                 die "skip\n";
4410                 # since we cannot reliably hot unplug usb devices we are disabling it
4411                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4412                 #vm_deviceunplug($vmid, $conf, $opt);
4413             } elsif ($opt eq 'vcpus') {
4414                 die "skip\n" if !$hotplug_features->{cpu};
4415                 qemu_cpu_hotplug($vmid, $conf, undef);
4416             } elsif ($opt eq 'balloon') {
4417                 # enable balloon device is not hotpluggable
4418                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4419                 # here we reset the ballooning value to memory
4420                 my $balloon = $conf->{memory} || $defaults->{memory};
4421                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4422             } elsif ($fast_plug_option->{$opt}) {
4423                 # do nothing
4424             } elsif ($opt =~ m/^net(\d+)$/) {
4425                 die "skip\n" if !$hotplug_features->{network};
4426                 vm_deviceunplug($vmid, $conf, $opt);
4427             } elsif (is_valid_drivename($opt)) {
4428                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4429                 vm_deviceunplug($vmid, $conf, $opt);
4430                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4431             } elsif ($opt =~ m/^memory$/) {
4432                 die "skip\n" if !$hotplug_features->{memory};
4433                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4434             } elsif ($opt eq 'cpuunits') {
4435                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4436             } elsif ($opt eq 'cpulimit') {
4437                 $cgroup->change_cpu_quota(-1, 100000);
4438             } else {
4439                 die "skip\n";
4440             }
4441         };
4442         if (my $err = $@) {
4443             &$add_error($opt, $err) if $err ne "skip\n";
4444         } else {
4445             delete $conf->{$opt};
4446             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4447         }
4448     }
4449
4450     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4451     $apply_pending_cloudinit = sub {
4452         return if $apply_pending_cloudinit_done; # once is enough
4453         $apply_pending_cloudinit_done = 1; # once is enough
4454
4455         my ($key, $value) = @_;
4456
4457         my @cloudinit_opts = keys %$confdesc_cloudinit;
4458         foreach my $opt (keys %{$conf->{pending}}) {
4459             next if !grep { $_ eq $opt } @cloudinit_opts;
4460             $conf->{$opt} = delete $conf->{pending}->{$opt};
4461         }
4462
4463         my $new_conf = { %$conf };
4464         $new_conf->{$key} = $value;
4465         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4466     };
4467
4468     foreach my $opt (keys %{$conf->{pending}}) {
4469         next if $selection && !$selection->{$opt};
4470         my $value = $conf->{pending}->{$opt};
4471         eval {
4472             if ($opt eq 'hotplug') {
4473                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4474             } elsif ($opt eq 'tablet') {
4475                 die "skip\n" if !$hotplug_features->{usb};
4476                 if ($value == 1) {
4477                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4478                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4479                         if $arch eq 'aarch64';
4480                 } elsif ($value == 0) {
4481                     vm_deviceunplug($vmid, $conf, 'tablet');
4482                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4483                 }
4484             } elsif ($opt =~ m/^usb\d+$/) {
4485                 die "skip\n";
4486                 # since we cannot reliably hot unplug usb devices we disable it for now
4487                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4488                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4489                 #die "skip\n" if !$d;
4490                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4491             } elsif ($opt eq 'vcpus') {
4492                 die "skip\n" if !$hotplug_features->{cpu};
4493                 qemu_cpu_hotplug($vmid, $conf, $value);
4494             } elsif ($opt eq 'balloon') {
4495                 # enable/disable balloning device is not hotpluggable
4496                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4497                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4498                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4499
4500                 # allow manual ballooning if shares is set to zero
4501                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4502                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4503                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4504                 }
4505             } elsif ($opt =~ m/^net(\d+)$/) {
4506                 # some changes can be done without hotplug
4507                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4508                                     $vmid, $opt, $value, $arch, $machine_type);
4509             } elsif (is_valid_drivename($opt)) {
4510                 die "skip\n" if $opt eq 'efidisk0';
4511                 # some changes can be done without hotplug
4512                 my $drive = parse_drive($opt, $value);
4513                 if (drive_is_cloudinit($drive)) {
4514                     &$apply_pending_cloudinit($opt, $value);
4515                 }
4516                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4517                                      $vmid, $opt, $value, $arch, $machine_type);
4518             } elsif ($opt =~ m/^memory$/) { #dimms
4519                 die "skip\n" if !$hotplug_features->{memory};
4520                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4521             } elsif ($opt eq 'cpuunits') {
4522                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4523             } elsif ($opt eq 'cpulimit') {
4524                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4525                 $cgroup->change_cpu_quota($cpulimit, 100000);
4526             } else {
4527                 die "skip\n";  # skip non-hot-pluggable options
4528             }
4529         };
4530         if (my $err = $@) {
4531             &$add_error($opt, $err) if $err ne "skip\n";
4532         } else {
4533             $conf->{$opt} = $value;
4534             delete $conf->{pending}->{$opt};
4535         }
4536     }
4537
4538     PVE::QemuConfig->write_config($vmid, $conf);
4539 }
4540
4541 sub try_deallocate_drive {
4542     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4543
4544     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4545         my $volid = $drive->{file};
4546         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4547             my $sid = PVE::Storage::parse_volume_id($volid);
4548             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4549
4550             # check if the disk is really unused
4551             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4552                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4553             PVE::Storage::vdisk_free($storecfg, $volid);
4554             return 1;
4555         } else {
4556             # If vm is not owner of this disk remove from config
4557             return 1;
4558         }
4559     }
4560
4561     return;
4562 }
4563
4564 sub vmconfig_delete_or_detach_drive {
4565     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4566
4567     my $drive = parse_drive($opt, $conf->{$opt});
4568
4569     my $rpcenv = PVE::RPCEnvironment::get();
4570     my $authuser = $rpcenv->get_user();
4571
4572     if ($force) {
4573         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4574         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4575     } else {
4576         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4577     }
4578 }
4579
4580
4581
4582 sub vmconfig_apply_pending {
4583     my ($vmid, $conf, $storecfg, $errors) = @_;
4584
4585     my $add_apply_error = sub {
4586         my ($opt, $msg) = @_;
4587         my $err_msg = "unable to apply pending change $opt : $msg";
4588         $errors->{$opt} = $err_msg;
4589         warn $err_msg;
4590     };
4591
4592     # cold plug
4593
4594     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4595     foreach my $opt (sort keys %$pending_delete_hash) {
4596         my $force = $pending_delete_hash->{$opt}->{force};
4597         eval {
4598             if ($opt =~ m/^unused/) {
4599                 die "internal error";
4600             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4601                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4602             }
4603         };
4604         if (my $err = $@) {
4605             $add_apply_error->($opt, $err);
4606         } else {
4607             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4608             delete $conf->{$opt};
4609         }
4610     }
4611
4612     PVE::QemuConfig->cleanup_pending($conf);
4613
4614     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4615         next if $opt eq 'delete'; # just to be sure
4616         eval {
4617             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4618                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4619             }
4620         };
4621         if (my $err = $@) {
4622             $add_apply_error->($opt, $err);
4623         } else {
4624             $conf->{$opt} = delete $conf->{pending}->{$opt};
4625         }
4626     }
4627
4628     # write all changes at once to avoid unnecessary i/o
4629     PVE::QemuConfig->write_config($vmid, $conf);
4630 }
4631
4632 sub vmconfig_update_net {
4633     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4634
4635     my $newnet = parse_net($value);
4636
4637     if ($conf->{$opt}) {
4638         my $oldnet = parse_net($conf->{$opt});
4639
4640         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4641             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4642             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4643             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4644
4645             # for non online change, we try to hot-unplug
4646             die "skip\n" if !$hotplug;
4647             vm_deviceunplug($vmid, $conf, $opt);
4648         } else {
4649
4650             die "internal error" if $opt !~ m/net(\d+)/;
4651             my $iface = "tap${vmid}i$1";
4652
4653             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4654                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4655                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4656                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4657                 PVE::Network::tap_unplug($iface);
4658
4659                 if ($have_sdn) {
4660                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4661                 } else {
4662                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4663                 }
4664             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4665                 # Rate can be applied on its own but any change above needs to
4666                 # include the rate in tap_plug since OVS resets everything.
4667                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4668             }
4669
4670             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4671                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4672             }
4673
4674             return 1;
4675         }
4676     }
4677
4678     if ($hotplug) {
4679         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4680     } else {
4681         die "skip\n";
4682     }
4683 }
4684
4685 sub vmconfig_update_disk {
4686     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4687
4688     my $drive = parse_drive($opt, $value);
4689
4690     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4691         my $media = $drive->{media} || 'disk';
4692         my $oldmedia = $old_drive->{media} || 'disk';
4693         die "unable to change media type\n" if $media ne $oldmedia;
4694
4695         if (!drive_is_cdrom($old_drive)) {
4696
4697             if ($drive->{file} ne $old_drive->{file}) {
4698
4699                 die "skip\n" if !$hotplug;
4700
4701                 # unplug and register as unused
4702                 vm_deviceunplug($vmid, $conf, $opt);
4703                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4704
4705             } else {
4706                 # update existing disk
4707
4708                 # skip non hotpluggable value
4709                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4710                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4711                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4712                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4713                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4714                     die "skip\n";
4715                 }
4716
4717                 # apply throttle
4718                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4719                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4720                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4721                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4722                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4723                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4724                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4725                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4726                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4727                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4728                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4729                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4730                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4731                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4732                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4733                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4734                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4735                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4736
4737                     qemu_block_set_io_throttle(
4738                         $vmid,"drive-$opt",
4739                         ($drive->{mbps} || 0)*1024*1024,
4740                         ($drive->{mbps_rd} || 0)*1024*1024,
4741                         ($drive->{mbps_wr} || 0)*1024*1024,
4742                          $drive->{iops} || 0,
4743                          $drive->{iops_rd} || 0,
4744                          $drive->{iops_wr} || 0,
4745                         ($drive->{mbps_max} || 0)*1024*1024,
4746                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4747                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4748                          $drive->{iops_max} || 0,
4749                          $drive->{iops_rd_max} || 0,
4750                          $drive->{iops_wr_max} || 0,
4751                          $drive->{bps_max_length} || 1,
4752                          $drive->{bps_rd_max_length} || 1,
4753                          $drive->{bps_wr_max_length} || 1,
4754                          $drive->{iops_max_length} || 1,
4755                          $drive->{iops_rd_max_length} || 1,
4756                          $drive->{iops_wr_max_length} || 1,
4757                     );
4758
4759                 }
4760
4761                 return 1;
4762             }
4763
4764         } else { # cdrom
4765
4766             if ($drive->{file} eq 'none') {
4767                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4768                 if (drive_is_cloudinit($old_drive)) {
4769                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4770                 }
4771             } else {
4772                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4773
4774                 # force eject if locked
4775                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4776
4777                 if ($path) {
4778                     mon_cmd($vmid, "blockdev-change-medium",
4779                         id => "$opt", filename => "$path");
4780                 }
4781             }
4782
4783             return 1;
4784         }
4785     }
4786
4787     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4788     # hotplug new disks
4789     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4790     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4791 }
4792
4793 # called in locked context by incoming migration
4794 sub vm_migrate_get_nbd_disks {
4795     my ($storecfg, $conf, $replicated_volumes) = @_;
4796
4797     my $local_volumes = {};
4798     PVE::QemuConfig->foreach_volume($conf, sub {
4799         my ($ds, $drive) = @_;
4800
4801         return if drive_is_cdrom($drive);
4802
4803         my $volid = $drive->{file};
4804
4805         return if !$volid;
4806
4807         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4808
4809         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4810         return if $scfg->{shared};
4811
4812         # replicated disks re-use existing state via bitmap
4813         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4814         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4815     });
4816     return $local_volumes;
4817 }
4818
4819 # called in locked context by incoming migration
4820 sub vm_migrate_alloc_nbd_disks {
4821     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4822
4823     my $format = undef;
4824
4825     my $nbd = {};
4826     foreach my $opt (sort keys %$source_volumes) {
4827         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4828
4829         if ($use_existing) {
4830             $nbd->{$opt}->{drivestr} = print_drive($drive);
4831             $nbd->{$opt}->{volid} = $volid;
4832             $nbd->{$opt}->{replicated} = 1;
4833             next;
4834         }
4835
4836         # If a remote storage is specified and the format of the original
4837         # volume is not available there, fall back to the default format.
4838         # Otherwise use the same format as the original.
4839         if (!$storagemap->{identity}) {
4840             $storeid = map_storage($storagemap, $storeid);
4841             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4842             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4843             my $fileFormat = qemu_img_format($scfg, $volname);
4844             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4845         } else {
4846             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4847             $format = qemu_img_format($scfg, $volname);
4848         }
4849
4850         my $size = $drive->{size} / 1024;
4851         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4852         my $newdrive = $drive;
4853         $newdrive->{format} = $format;
4854         $newdrive->{file} = $newvolid;
4855         my $drivestr = print_drive($newdrive);
4856         $nbd->{$opt}->{drivestr} = $drivestr;
4857         $nbd->{$opt}->{volid} = $newvolid;
4858     }
4859
4860     return $nbd;
4861 }
4862
4863 # see vm_start_nolock for parameters, additionally:
4864 # migrate_opts:
4865 #   storagemap = parsed storage map for allocating NBD disks
4866 sub vm_start {
4867     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4868
4869     return PVE::QemuConfig->lock_config($vmid, sub {
4870         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4871
4872         die "you can't start a vm if it's a template\n"
4873             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4874
4875         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4876         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4877
4878         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4879
4880         if ($has_backup_lock && $running) {
4881             # a backup is currently running, attempt to start the guest in the
4882             # existing QEMU instance
4883             return vm_resume($vmid);
4884         }
4885
4886         PVE::QemuConfig->check_lock($conf)
4887             if !($params->{skiplock} || $has_suspended_lock);
4888
4889         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4890
4891         die "VM $vmid already running\n" if $running;
4892
4893         if (my $storagemap = $migrate_opts->{storagemap}) {
4894             my $replicated = $migrate_opts->{replicated_volumes};
4895             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4896             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4897
4898             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4899                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4900             }
4901         }
4902
4903         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4904     });
4905 }
4906
4907
4908 # params:
4909 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4910 #   skiplock => 0/1, skip checking for config lock
4911 #   skiptemplate => 0/1, skip checking whether VM is template
4912 #   forcemachine => to force Qemu machine (rollback/migration)
4913 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4914 #   timeout => in seconds
4915 #   paused => start VM in paused state (backup)
4916 #   resume => resume from hibernation
4917 # migrate_opts:
4918 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4919 #   migratedfrom => source node
4920 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4921 #   network => CIDR of migration network
4922 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4923 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4924 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4925 sub vm_start_nolock {
4926     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4927
4928     my $statefile = $params->{statefile};
4929     my $resume = $params->{resume};
4930
4931     my $migratedfrom = $migrate_opts->{migratedfrom};
4932     my $migration_type = $migrate_opts->{type};
4933
4934     my $res = {};
4935
4936     # clean up leftover reboot request files
4937     eval { clear_reboot_request($vmid); };
4938     warn $@ if $@;
4939
4940     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4941         vmconfig_apply_pending($vmid, $conf, $storecfg);
4942         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4943     }
4944
4945     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4946
4947     my $defaults = load_defaults();
4948
4949     # set environment variable useful inside network script
4950     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4951
4952     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4953
4954     my $forcemachine = $params->{forcemachine};
4955     my $forcecpu = $params->{forcecpu};
4956     if ($resume) {
4957         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4958         $forcemachine = $conf->{runningmachine};
4959         $forcecpu = $conf->{runningcpu};
4960         print "Resuming suspended VM\n";
4961     }
4962
4963     my ($cmd, $vollist, $spice_port) =
4964         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4965
4966     my $migration_ip;
4967     my $get_migration_ip = sub {
4968         my ($nodename) = @_;
4969
4970         return $migration_ip if defined($migration_ip);
4971
4972         my $cidr = $migrate_opts->{network};
4973
4974         if (!defined($cidr)) {
4975             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4976             $cidr = $dc_conf->{migration}->{network};
4977         }
4978
4979         if (defined($cidr)) {
4980             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4981
4982             die "could not get IP: no address configured on local " .
4983                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4984
4985             die "could not get IP: multiple addresses configured on local " .
4986                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4987
4988             $migration_ip = @$ips[0];
4989         }
4990
4991         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4992             if !defined($migration_ip);
4993
4994         return $migration_ip;
4995     };
4996
4997     my $migrate_uri;
4998     if ($statefile) {
4999         if ($statefile eq 'tcp') {
5000             my $localip = "localhost";
5001             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5002             my $nodename = nodename();
5003
5004             if (!defined($migration_type)) {
5005                 if (defined($datacenterconf->{migration}->{type})) {
5006                     $migration_type = $datacenterconf->{migration}->{type};
5007                 } else {
5008                     $migration_type = 'secure';
5009                 }
5010             }
5011
5012             if ($migration_type eq 'insecure') {
5013                 $localip = $get_migration_ip->($nodename);
5014                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5015             }
5016
5017             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5018             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5019             $migrate_uri = "tcp:${localip}:${migrate_port}";
5020             push @$cmd, '-incoming', $migrate_uri;
5021             push @$cmd, '-S';
5022
5023         } elsif ($statefile eq 'unix') {
5024             # should be default for secure migrations as a ssh TCP forward
5025             # tunnel is not deterministic reliable ready and fails regurarly
5026             # to set up in time, so use UNIX socket forwards
5027             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5028             unlink $socket_addr;
5029
5030             $migrate_uri = "unix:$socket_addr";
5031
5032             push @$cmd, '-incoming', $migrate_uri;
5033             push @$cmd, '-S';
5034
5035         } elsif (-e $statefile) {
5036             push @$cmd, '-loadstate', $statefile;
5037         } else {
5038             my $statepath = PVE::Storage::path($storecfg, $statefile);
5039             push @$vollist, $statefile;
5040             push @$cmd, '-loadstate', $statepath;
5041         }
5042     } elsif ($params->{paused}) {
5043         push @$cmd, '-S';
5044     }
5045
5046     # host pci devices
5047     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5048       my $d = parse_hostpci($conf->{"hostpci$i"});
5049       next if !$d;
5050       my $pcidevices = $d->{pciid};
5051       foreach my $pcidevice (@$pcidevices) {
5052             my $pciid = $pcidevice->{id};
5053
5054             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5055             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5056             die "no pci device info for device '$pciid'\n" if !$info;
5057
5058             if ($d->{mdev}) {
5059                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5060                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5061             } else {
5062                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5063                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5064                 die "can't reset PCI device '$pciid'\n"
5065                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5066             }
5067       }
5068     }
5069
5070     PVE::Storage::activate_volumes($storecfg, $vollist);
5071
5072     eval {
5073         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5074             outfunc => sub {}, errfunc => sub {});
5075     };
5076     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5077     # timeout should be more than enough here...
5078     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5079
5080     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5081                                               : $defaults->{cpuunits};
5082
5083     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5084     my %run_params = (
5085         timeout => $statefile ? undef : $start_timeout,
5086         umask => 0077,
5087         noerr => 1,
5088     );
5089
5090     # when migrating, prefix QEMU output so other side can pick up any
5091     # errors that might occur and show the user
5092     if ($migratedfrom) {
5093         $run_params{quiet} = 1;
5094         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5095     }
5096
5097     my %properties = (
5098         Slice => 'qemu.slice',
5099         KillMode => 'none'
5100     );
5101
5102     if (PVE::CGroup::cgroup_mode() == 2) {
5103         $properties{CPUWeight} = $cpuunits;
5104     } else {
5105         $properties{CPUShares} = $cpuunits;
5106     }
5107
5108     if (my $cpulimit = $conf->{cpulimit}) {
5109         $properties{CPUQuota} = int($cpulimit * 100);
5110     }
5111     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5112
5113     my $run_qemu = sub {
5114         PVE::Tools::run_fork sub {
5115             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5116
5117             my $exitcode = run_command($cmd, %run_params);
5118             die "QEMU exited with code $exitcode\n" if $exitcode;
5119         };
5120     };
5121
5122     if ($conf->{hugepages}) {
5123
5124         my $code = sub {
5125             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5126             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5127
5128             PVE::QemuServer::Memory::hugepages_mount();
5129             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5130
5131             eval { $run_qemu->() };
5132             if (my $err = $@) {
5133                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5134                     if !$conf->{keephugepages};
5135                 die $err;
5136             }
5137
5138             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5139                 if !$conf->{keephugepages};
5140         };
5141         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5142
5143     } else {
5144         eval { $run_qemu->() };
5145     }
5146
5147     if (my $err = $@) {
5148         # deactivate volumes if start fails
5149         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5150         die "start failed: $err";
5151     }
5152
5153     print "migration listens on $migrate_uri\n" if $migrate_uri;
5154     $res->{migrate_uri} = $migrate_uri;
5155
5156     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5157         eval { mon_cmd($vmid, "cont"); };
5158         warn $@ if $@;
5159     }
5160
5161     #start nbd server for storage migration
5162     if (my $nbd = $migrate_opts->{nbd}) {
5163         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5164
5165         my $migrate_storage_uri;
5166         # nbd_protocol_version > 0 for unix socket support
5167         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5168             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5169             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5170             $migrate_storage_uri = "nbd:unix:$socket_path";
5171         } else {
5172             my $nodename = nodename();
5173             my $localip = $get_migration_ip->($nodename);
5174             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5175             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5176
5177             mon_cmd($vmid, "nbd-server-start", addr => {
5178                 type => 'inet',
5179                 data => {
5180                     host => "${localip}",
5181                     port => "${storage_migrate_port}",
5182                 },
5183             });
5184             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5185             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5186         }
5187
5188         $res->{migrate_storage_uri} = $migrate_storage_uri;
5189
5190         foreach my $opt (sort keys %$nbd) {
5191             my $drivestr = $nbd->{$opt}->{drivestr};
5192             my $volid = $nbd->{$opt}->{volid};
5193             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5194             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5195             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5196             print "re-using replicated volume: $opt - $volid\n"
5197                 if $nbd->{$opt}->{replicated};
5198
5199             $res->{drives}->{$opt} = $nbd->{$opt};
5200             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5201         }
5202     }
5203
5204     if ($migratedfrom) {
5205         eval {
5206             set_migration_caps($vmid);
5207         };
5208         warn $@ if $@;
5209
5210         if ($spice_port) {
5211             print "spice listens on port $spice_port\n";
5212             $res->{spice_port} = $spice_port;
5213             if ($migrate_opts->{spice_ticket}) {
5214                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5215                     $migrate_opts->{spice_ticket});
5216                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5217             }
5218         }
5219
5220     } else {
5221         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5222             if !$statefile && $conf->{balloon};
5223
5224         foreach my $opt (keys %$conf) {
5225             next if $opt !~  m/^net\d+$/;
5226             my $nicconf = parse_net($conf->{$opt});
5227             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5228         }
5229     }
5230
5231     mon_cmd($vmid, 'qom-set',
5232                 path => "machine/peripheral/balloon0",
5233                 property => "guest-stats-polling-interval",
5234                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5235
5236     if ($resume) {
5237         print "Resumed VM, removing state\n";
5238         if (my $vmstate = $conf->{vmstate}) {
5239             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5240             PVE::Storage::vdisk_free($storecfg, $vmstate);
5241         }
5242         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5243         PVE::QemuConfig->write_config($vmid, $conf);
5244     }
5245
5246     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5247
5248     return $res;
5249 }
5250
5251 sub vm_commandline {
5252     my ($storecfg, $vmid, $snapname) = @_;
5253
5254     my $conf = PVE::QemuConfig->load_config($vmid);
5255     my $forcemachine;
5256     my $forcecpu;
5257
5258     if ($snapname) {
5259         my $snapshot = $conf->{snapshots}->{$snapname};
5260         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5261
5262         # check for machine or CPU overrides in snapshot
5263         $forcemachine = $snapshot->{runningmachine};
5264         $forcecpu = $snapshot->{runningcpu};
5265
5266         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5267
5268         $conf = $snapshot;
5269     }
5270
5271     my $defaults = load_defaults();
5272
5273     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5274         $forcemachine, $forcecpu);
5275
5276     return PVE::Tools::cmd2string($cmd);
5277 }
5278
5279 sub vm_reset {
5280     my ($vmid, $skiplock) = @_;
5281
5282     PVE::QemuConfig->lock_config($vmid, sub {
5283
5284         my $conf = PVE::QemuConfig->load_config($vmid);
5285
5286         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5287
5288         mon_cmd($vmid, "system_reset");
5289     });
5290 }
5291
5292 sub get_vm_volumes {
5293     my ($conf) = @_;
5294
5295     my $vollist = [];
5296     foreach_volid($conf, sub {
5297         my ($volid, $attr) = @_;
5298
5299         return if $volid =~ m|^/|;
5300
5301         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5302         return if !$sid;
5303
5304         push @$vollist, $volid;
5305     });
5306
5307     return $vollist;
5308 }
5309
5310 sub vm_stop_cleanup {
5311     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5312
5313     eval {
5314
5315         if (!$keepActive) {
5316             my $vollist = get_vm_volumes($conf);
5317             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5318         }
5319
5320         foreach my $ext (qw(mon qmp pid vnc qga)) {
5321             unlink "/var/run/qemu-server/${vmid}.$ext";
5322         }
5323
5324         if ($conf->{ivshmem}) {
5325             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5326             # just delete it for now, VMs which have this already open do not
5327             # are affected, but new VMs will get a separated one. If this
5328             # becomes an issue we either add some sort of ref-counting or just
5329             # add a "don't delete on stop" flag to the ivshmem format.
5330             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5331         }
5332
5333         foreach my $key (keys %$conf) {
5334             next if $key !~ m/^hostpci(\d+)$/;
5335             my $hostpciindex = $1;
5336             my $d = parse_hostpci($conf->{$key});
5337             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5338
5339             foreach my $pci (@{$d->{pciid}}) {
5340                 my $pciid = $pci->{id};
5341                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5342             }
5343         }
5344
5345         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5346     };
5347     warn $@ if $@; # avoid errors - just warn
5348 }
5349
5350 # call only in locked context
5351 sub _do_vm_stop {
5352     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5353
5354     my $pid = check_running($vmid, $nocheck);
5355     return if !$pid;
5356
5357     my $conf;
5358     if (!$nocheck) {
5359         $conf = PVE::QemuConfig->load_config($vmid);
5360         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5361         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5362             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5363             $timeout = $opts->{down} if $opts->{down};
5364         }
5365         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5366     }
5367
5368     eval {
5369         if ($shutdown) {
5370             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5371                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5372             } else {
5373                 mon_cmd($vmid, "system_powerdown");
5374             }
5375         } else {
5376             mon_cmd($vmid, "quit");
5377         }
5378     };
5379     my $err = $@;
5380
5381     if (!$err) {
5382         $timeout = 60 if !defined($timeout);
5383
5384         my $count = 0;
5385         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5386             $count++;
5387             sleep 1;
5388         }
5389
5390         if ($count >= $timeout) {
5391             if ($force) {
5392                 warn "VM still running - terminating now with SIGTERM\n";
5393                 kill 15, $pid;
5394             } else {
5395                 die "VM quit/powerdown failed - got timeout\n";
5396             }
5397         } else {
5398             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5399             return;
5400         }
5401     } else {
5402        if (!check_running($vmid, $nocheck)) {
5403             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5404             return;
5405        }
5406        if ($force) {
5407             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5408             kill 15, $pid;
5409         } else {
5410             die "VM quit/powerdown failed\n";
5411         }
5412     }
5413
5414     # wait again
5415     $timeout = 10;
5416
5417     my $count = 0;
5418     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5419         $count++;
5420         sleep 1;
5421     }
5422
5423     if ($count >= $timeout) {
5424         warn "VM still running - terminating now with SIGKILL\n";
5425         kill 9, $pid;
5426         sleep 1;
5427     }
5428
5429     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5430 }
5431
5432 # Note: use $nocheck to skip tests if VM configuration file exists.
5433 # We need that when migration VMs to other nodes (files already moved)
5434 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5435 sub vm_stop {
5436     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5437
5438     $force = 1 if !defined($force) && !$shutdown;
5439
5440     if ($migratedfrom){
5441         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5442         kill 15, $pid if $pid;
5443         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5444         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5445         return;
5446     }
5447
5448     PVE::QemuConfig->lock_config($vmid, sub {
5449         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5450    });
5451 }
5452
5453 sub vm_reboot {
5454     my ($vmid, $timeout) = @_;
5455
5456     PVE::QemuConfig->lock_config($vmid, sub {
5457         eval {
5458
5459             # only reboot if running, as qmeventd starts it again on a stop event
5460             return if !check_running($vmid);
5461
5462             create_reboot_request($vmid);
5463
5464             my $storecfg = PVE::Storage::config();
5465             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5466
5467         };
5468         if (my $err = $@) {
5469             # avoid that the next normal shutdown will be confused for a reboot
5470             clear_reboot_request($vmid);
5471             die $err;
5472         }
5473    });
5474 }
5475
5476 # note: if using the statestorage parameter, the caller has to check privileges
5477 sub vm_suspend {
5478     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5479
5480     my $conf;
5481     my $path;
5482     my $storecfg;
5483     my $vmstate;
5484
5485     PVE::QemuConfig->lock_config($vmid, sub {
5486
5487         $conf = PVE::QemuConfig->load_config($vmid);
5488
5489         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5490         PVE::QemuConfig->check_lock($conf)
5491             if !($skiplock || $is_backing_up);
5492
5493         die "cannot suspend to disk during backup\n"
5494             if $is_backing_up && $includestate;
5495
5496         if ($includestate) {
5497             $conf->{lock} = 'suspending';
5498             my $date = strftime("%Y-%m-%d", localtime(time()));
5499             $storecfg = PVE::Storage::config();
5500             if (!$statestorage) {
5501                 $statestorage = find_vmstate_storage($conf, $storecfg);
5502                 # check permissions for the storage
5503                 my $rpcenv = PVE::RPCEnvironment::get();
5504                 if ($rpcenv->{type} ne 'cli') {
5505                     my $authuser = $rpcenv->get_user();
5506                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5507                 }
5508             }
5509
5510
5511             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5512                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5513             $path = PVE::Storage::path($storecfg, $vmstate);
5514             PVE::QemuConfig->write_config($vmid, $conf);
5515         } else {
5516             mon_cmd($vmid, "stop");
5517         }
5518     });
5519
5520     if ($includestate) {
5521         # save vm state
5522         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5523
5524         eval {
5525             mon_cmd($vmid, "savevm-start", statefile => $path);
5526             for(;;) {
5527                 my $state = mon_cmd($vmid, "query-savevm");
5528                 if (!$state->{status}) {
5529                     die "savevm not active\n";
5530                 } elsif ($state->{status} eq 'active') {
5531                     sleep(1);
5532                     next;
5533                 } elsif ($state->{status} eq 'completed') {
5534                     print "State saved, quitting\n";
5535                     last;
5536                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5537                     die "query-savevm failed with error '$state->{error}'\n"
5538                 } else {
5539                     die "query-savevm returned status '$state->{status}'\n";
5540                 }
5541             }
5542         };
5543         my $err = $@;
5544
5545         PVE::QemuConfig->lock_config($vmid, sub {
5546             $conf = PVE::QemuConfig->load_config($vmid);
5547             if ($err) {
5548                 # cleanup, but leave suspending lock, to indicate something went wrong
5549                 eval {
5550                     mon_cmd($vmid, "savevm-end");
5551                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5552                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5553                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5554                     PVE::QemuConfig->write_config($vmid, $conf);
5555                 };
5556                 warn $@ if $@;
5557                 die $err;
5558             }
5559
5560             die "lock changed unexpectedly\n"
5561                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5562
5563             mon_cmd($vmid, "quit");
5564             $conf->{lock} = 'suspended';
5565             PVE::QemuConfig->write_config($vmid, $conf);
5566         });
5567     }
5568 }
5569
5570 sub vm_resume {
5571     my ($vmid, $skiplock, $nocheck) = @_;
5572
5573     PVE::QemuConfig->lock_config($vmid, sub {
5574         my $res = mon_cmd($vmid, 'query-status');
5575         my $resume_cmd = 'cont';
5576         my $reset = 0;
5577
5578         if ($res->{status}) {
5579             return if $res->{status} eq 'running'; # job done, go home
5580             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5581             $reset = 1 if $res->{status} eq 'shutdown';
5582         }
5583
5584         if (!$nocheck) {
5585
5586             my $conf = PVE::QemuConfig->load_config($vmid);
5587
5588             PVE::QemuConfig->check_lock($conf)
5589                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5590         }
5591
5592         if ($reset) {
5593             # required if a VM shuts down during a backup and we get a resume
5594             # request before the backup finishes for example
5595             mon_cmd($vmid, "system_reset");
5596         }
5597         mon_cmd($vmid, $resume_cmd);
5598     });
5599 }
5600
5601 sub vm_sendkey {
5602     my ($vmid, $skiplock, $key) = @_;
5603
5604     PVE::QemuConfig->lock_config($vmid, sub {
5605
5606         my $conf = PVE::QemuConfig->load_config($vmid);
5607
5608         # there is no qmp command, so we use the human monitor command
5609         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5610         die $res if $res ne '';
5611     });
5612 }
5613
5614 # vzdump restore implementaion
5615
5616 sub tar_archive_read_firstfile {
5617     my $archive = shift;
5618
5619     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5620
5621     # try to detect archive type first
5622     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5623         die "unable to open file '$archive'\n";
5624     my $firstfile = <$fh>;
5625     kill 15, $pid;
5626     close $fh;
5627
5628     die "ERROR: archive contaions no data\n" if !$firstfile;
5629     chomp $firstfile;
5630
5631     return $firstfile;
5632 }
5633
5634 sub tar_restore_cleanup {
5635     my ($storecfg, $statfile) = @_;
5636
5637     print STDERR "starting cleanup\n";
5638
5639     if (my $fd = IO::File->new($statfile, "r")) {
5640         while (defined(my $line = <$fd>)) {
5641             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5642                 my $volid = $2;
5643                 eval {
5644                     if ($volid =~ m|^/|) {
5645                         unlink $volid || die 'unlink failed\n';
5646                     } else {
5647                         PVE::Storage::vdisk_free($storecfg, $volid);
5648                     }
5649                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5650                 };
5651                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5652             } else {
5653                 print STDERR "unable to parse line in statfile - $line";
5654             }
5655         }
5656         $fd->close();
5657     }
5658 }
5659
5660 sub restore_file_archive {
5661     my ($archive, $vmid, $user, $opts) = @_;
5662
5663     return restore_vma_archive($archive, $vmid, $user, $opts)
5664         if $archive eq '-';
5665
5666     my $info = PVE::Storage::archive_info($archive);
5667     my $format = $opts->{format} // $info->{format};
5668     my $comp = $info->{compression};
5669
5670     # try to detect archive format
5671     if ($format eq 'tar') {
5672         return restore_tar_archive($archive, $vmid, $user, $opts);
5673     } else {
5674         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5675     }
5676 }
5677
5678 # hepler to remove disks that will not be used after restore
5679 my $restore_cleanup_oldconf = sub {
5680     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5681
5682     PVE::QemuConfig->foreach_volume($oldconf, sub {
5683         my ($ds, $drive) = @_;
5684
5685         return if drive_is_cdrom($drive, 1);
5686
5687         my $volid = $drive->{file};
5688         return if !$volid || $volid =~ m|^/|;
5689
5690         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5691         return if !$path || !$owner || ($owner != $vmid);
5692
5693         # Note: only delete disk we want to restore
5694         # other volumes will become unused
5695         if ($virtdev_hash->{$ds}) {
5696             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5697             if (my $err = $@) {
5698                 warn $err;
5699             }
5700         }
5701     });
5702
5703     # delete vmstate files, after the restore we have no snapshots anymore
5704     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5705         my $snap = $oldconf->{snapshots}->{$snapname};
5706         if ($snap->{vmstate}) {
5707             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5708             if (my $err = $@) {
5709                 warn $err;
5710             }
5711         }
5712     }
5713 };
5714
5715 # Helper to parse vzdump backup device hints
5716 #
5717 # $rpcenv: Environment, used to ckeck storage permissions
5718 # $user: User ID, to check storage permissions
5719 # $storecfg: Storage configuration
5720 # $fh: the file handle for reading the configuration
5721 # $devinfo: should contain device sizes for all backu-up'ed devices
5722 # $options: backup options (pool, default storage)
5723 #
5724 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5725 my $parse_backup_hints = sub {
5726     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5727
5728     my $virtdev_hash = {};
5729
5730     while (defined(my $line = <$fh>)) {
5731         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5732             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5733             die "archive does not contain data for drive '$virtdev'\n"
5734                 if !$devinfo->{$devname};
5735
5736             if (defined($options->{storage})) {
5737                 $storeid = $options->{storage} || 'local';
5738             } elsif (!$storeid) {
5739                 $storeid = 'local';
5740             }
5741             $format = 'raw' if !$format;
5742             $devinfo->{$devname}->{devname} = $devname;
5743             $devinfo->{$devname}->{virtdev} = $virtdev;
5744             $devinfo->{$devname}->{format} = $format;
5745             $devinfo->{$devname}->{storeid} = $storeid;
5746
5747             # check permission on storage
5748             my $pool = $options->{pool}; # todo: do we need that?
5749             if ($user ne 'root@pam') {
5750                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5751             }
5752
5753             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5754         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5755             my $virtdev = $1;
5756             my $drive = parse_drive($virtdev, $2);
5757             if (drive_is_cloudinit($drive)) {
5758                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5759                 $storeid = $options->{storage} if defined ($options->{storage});
5760                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5761                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5762
5763                 $virtdev_hash->{$virtdev} = {
5764                     format => $format,
5765                     storeid => $storeid,
5766                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5767                     is_cloudinit => 1,
5768                 };
5769             }
5770         }
5771     }
5772
5773     return $virtdev_hash;
5774 };
5775
5776 # Helper to allocate and activate all volumes required for a restore
5777 #
5778 # $storecfg: Storage configuration
5779 # $virtdev_hash: as returned by parse_backup_hints()
5780 #
5781 # Returns: { $virtdev => $volid }
5782 my $restore_allocate_devices = sub {
5783     my ($storecfg, $virtdev_hash, $vmid) = @_;
5784
5785     my $map = {};
5786     foreach my $virtdev (sort keys %$virtdev_hash) {
5787         my $d = $virtdev_hash->{$virtdev};
5788         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5789         my $storeid = $d->{storeid};
5790         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5791
5792         # test if requested format is supported
5793         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5794         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5795         $d->{format} = $defFormat if !$supported;
5796
5797         my $name;
5798         if ($d->{is_cloudinit}) {
5799             $name = "vm-$vmid-cloudinit";
5800             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5801         }
5802
5803         my $volid = PVE::Storage::vdisk_alloc(
5804             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5805
5806         print STDERR "new volume ID is '$volid'\n";
5807         $d->{volid} = $volid;
5808
5809         PVE::Storage::activate_volumes($storecfg, [$volid]);
5810
5811         $map->{$virtdev} = $volid;
5812     }
5813
5814     return $map;
5815 };
5816
5817 my $restore_update_config_line = sub {
5818     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5819
5820     return if $line =~ m/^\#qmdump\#/;
5821     return if $line =~ m/^\#vzdump\#/;
5822     return if $line =~ m/^lock:/;
5823     return if $line =~ m/^unused\d+:/;
5824     return if $line =~ m/^parent:/;
5825
5826     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5827     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5828         # try to convert old 1.X settings
5829         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5830         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5831             my ($model, $macaddr) = split(/\=/, $devconfig);
5832             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5833             my $net = {
5834                 model => $model,
5835                 bridge => "vmbr$ind",
5836                 macaddr => $macaddr,
5837             };
5838             my $netstr = print_net($net);
5839
5840             print $outfd "net$cookie->{netcount}: $netstr\n";
5841             $cookie->{netcount}++;
5842         }
5843     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5844         my ($id, $netstr) = ($1, $2);
5845         my $net = parse_net($netstr);
5846         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5847         $netstr = print_net($net);
5848         print $outfd "$id: $netstr\n";
5849     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5850         my $virtdev = $1;
5851         my $value = $3;
5852         my $di = parse_drive($virtdev, $value);
5853         if (defined($di->{backup}) && !$di->{backup}) {
5854             print $outfd "#$line";
5855         } elsif ($map->{$virtdev}) {
5856             delete $di->{format}; # format can change on restore
5857             $di->{file} = $map->{$virtdev};
5858             $value = print_drive($di);
5859             print $outfd "$virtdev: $value\n";
5860         } else {
5861             print $outfd $line;
5862         }
5863     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5864         my $vmgenid = $1;
5865         if ($vmgenid ne '0') {
5866             # always generate a new vmgenid if there was a valid one setup
5867             $vmgenid = generate_uuid();
5868         }
5869         print $outfd "vmgenid: $vmgenid\n";
5870     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5871         my ($uuid, $uuid_str);
5872         UUID::generate($uuid);
5873         UUID::unparse($uuid, $uuid_str);
5874         my $smbios1 = parse_smbios1($2);
5875         $smbios1->{uuid} = $uuid_str;
5876         print $outfd $1.print_smbios1($smbios1)."\n";
5877     } else {
5878         print $outfd $line;
5879     }
5880 };
5881
5882 my $restore_deactivate_volumes = sub {
5883     my ($storecfg, $devinfo) = @_;
5884
5885     my $vollist = [];
5886     foreach my $devname (keys %$devinfo) {
5887         my $volid = $devinfo->{$devname}->{volid};
5888         push @$vollist, $volid if $volid;
5889     }
5890
5891     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5892 };
5893
5894 my $restore_destroy_volumes = sub {
5895     my ($storecfg, $devinfo) = @_;
5896
5897     foreach my $devname (keys %$devinfo) {
5898         my $volid = $devinfo->{$devname}->{volid};
5899         next if !$volid;
5900         eval {
5901             if ($volid =~ m|^/|) {
5902                 unlink $volid || die 'unlink failed\n';
5903             } else {
5904                 PVE::Storage::vdisk_free($storecfg, $volid);
5905             }
5906             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5907         };
5908         print STDERR "unable to cleanup '$volid' - $@" if $@;
5909     }
5910 };
5911
5912 sub scan_volids {
5913     my ($cfg, $vmid) = @_;
5914
5915     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5916
5917     my $volid_hash = {};
5918     foreach my $storeid (keys %$info) {
5919         foreach my $item (@{$info->{$storeid}}) {
5920             next if !($item->{volid} && $item->{size});
5921             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5922             $volid_hash->{$item->{volid}} = $item;
5923         }
5924     }
5925
5926     return $volid_hash;
5927 }
5928
5929 sub update_disk_config {
5930     my ($vmid, $conf, $volid_hash) = @_;
5931
5932     my $changes;
5933     my $prefix = "VM $vmid";
5934
5935     # used and unused disks
5936     my $referenced = {};
5937
5938     # Note: it is allowed to define multiple storages with same path (alias), so
5939     # we need to check both 'volid' and real 'path' (two different volid can point
5940     # to the same path).
5941
5942     my $referencedpath = {};
5943
5944     # update size info
5945     PVE::QemuConfig->foreach_volume($conf, sub {
5946         my ($opt, $drive) = @_;
5947
5948         my $volid = $drive->{file};
5949         return if !$volid;
5950         my $volume = $volid_hash->{$volid};
5951
5952         # mark volid as "in-use" for next step
5953         $referenced->{$volid} = 1;
5954         if ($volume && (my $path = $volume->{path})) {
5955             $referencedpath->{$path} = 1;
5956         }
5957
5958         return if drive_is_cdrom($drive);
5959         return if !$volume;
5960
5961         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5962         if (defined($updated)) {
5963             $changes = 1;
5964             $conf->{$opt} = print_drive($updated);
5965             print "$prefix ($opt): $msg\n";
5966         }
5967     });
5968
5969     # remove 'unusedX' entry if volume is used
5970     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5971         my ($opt, $drive) = @_;
5972
5973         my $volid = $drive->{file};
5974         return if !$volid;
5975
5976         my $path;
5977         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5978         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5979             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5980             $changes = 1;
5981             delete $conf->{$opt};
5982         }
5983
5984         $referenced->{$volid} = 1;
5985         $referencedpath->{$path} = 1 if $path;
5986     });
5987
5988     foreach my $volid (sort keys %$volid_hash) {
5989         next if $volid =~ m/vm-$vmid-state-/;
5990         next if $referenced->{$volid};
5991         my $path = $volid_hash->{$volid}->{path};
5992         next if !$path; # just to be sure
5993         next if $referencedpath->{$path};
5994         $changes = 1;
5995         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5996         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5997         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5998     }
5999
6000     return $changes;
6001 }
6002
6003 sub rescan {
6004     my ($vmid, $nolock, $dryrun) = @_;
6005
6006     my $cfg = PVE::Storage::config();
6007
6008     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6009     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6010     foreach my $stor (keys %{$cfg->{ids}}) {
6011         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6012     }
6013
6014     print "rescan volumes...\n";
6015     my $volid_hash = scan_volids($cfg, $vmid);
6016
6017     my $updatefn =  sub {
6018         my ($vmid) = @_;
6019
6020         my $conf = PVE::QemuConfig->load_config($vmid);
6021
6022         PVE::QemuConfig->check_lock($conf);
6023
6024         my $vm_volids = {};
6025         foreach my $volid (keys %$volid_hash) {
6026             my $info = $volid_hash->{$volid};
6027             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6028         }
6029
6030         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6031
6032         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6033     };
6034
6035     if (defined($vmid)) {
6036         if ($nolock) {
6037             &$updatefn($vmid);
6038         } else {
6039             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6040         }
6041     } else {
6042         my $vmlist = config_list();
6043         foreach my $vmid (keys %$vmlist) {
6044             if ($nolock) {
6045                 &$updatefn($vmid);
6046             } else {
6047                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6048             }
6049         }
6050     }
6051 }
6052
6053 sub restore_proxmox_backup_archive {
6054     my ($archive, $vmid, $user, $options) = @_;
6055
6056     my $storecfg = PVE::Storage::config();
6057
6058     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6059     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6060
6061     my $server = $scfg->{server};
6062     my $datastore = $scfg->{datastore};
6063     my $username = $scfg->{username} // 'root@pam';
6064     my $fingerprint = $scfg->{fingerprint};
6065     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6066
6067     my $repo = "$username\@$server:$datastore";
6068
6069     # This is only used for `pbs-restore`!
6070     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6071     local $ENV{PBS_PASSWORD} = $password;
6072     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6073
6074     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6075         PVE::Storage::parse_volname($storecfg, $archive);
6076
6077     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6078
6079     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6080
6081     my $tmpdir = "/var/tmp/vzdumptmp$$";
6082     rmtree $tmpdir;
6083     mkpath $tmpdir;
6084
6085     my $conffile = PVE::QemuConfig->config_file($vmid);
6086     my $tmpfn = "$conffile.$$.tmp";
6087      # disable interrupts (always do cleanups)
6088     local $SIG{INT} =
6089         local $SIG{TERM} =
6090         local $SIG{QUIT} =
6091         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6092
6093     # Note: $oldconf is undef if VM does not exists
6094     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6095     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6096
6097     my $rpcenv = PVE::RPCEnvironment::get();
6098     my $devinfo = {};
6099
6100     eval {
6101         # enable interrupts
6102         local $SIG{INT} =
6103             local $SIG{TERM} =
6104             local $SIG{QUIT} =
6105             local $SIG{HUP} =
6106             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6107
6108         my $cfgfn = "$tmpdir/qemu-server.conf";
6109         my $firewall_config_fn = "$tmpdir/fw.conf";
6110         my $index_fn = "$tmpdir/index.json";
6111
6112         my $cmd = "restore";
6113
6114         my $param = [$pbs_backup_name, "index.json", $index_fn];
6115         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6116         my $index = PVE::Tools::file_get_contents($index_fn);
6117         $index = decode_json($index);
6118
6119         # print Dumper($index);
6120         foreach my $info (@{$index->{files}}) {
6121             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6122                 my $devname = $1;
6123                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6124                     $devinfo->{$devname}->{size} = $1;
6125                 } else {
6126                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6127                 }
6128             }
6129         }
6130
6131         my $is_qemu_server_backup = scalar(
6132             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6133         );
6134         if (!$is_qemu_server_backup) {
6135             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6136         }
6137         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6138
6139         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6140         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6141
6142         if ($has_firewall_config) {
6143             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6144             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6145
6146             my $pve_firewall_dir = '/etc/pve/firewall';
6147             mkdir $pve_firewall_dir; # make sure the dir exists
6148             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6149         }
6150
6151         my $fh = IO::File->new($cfgfn, "r") ||
6152             die "unable to read qemu-server.conf - $!\n";
6153
6154         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6155
6156         # fixme: rate limit?
6157
6158         # create empty/temp config
6159         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6160
6161         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6162
6163         # allocate volumes
6164         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6165
6166         foreach my $virtdev (sort keys %$virtdev_hash) {
6167             my $d = $virtdev_hash->{$virtdev};
6168             next if $d->{is_cloudinit}; # no need to restore cloudinit
6169
6170             my $volid = $d->{volid};
6171
6172             my $path = PVE::Storage::path($storecfg, $volid);
6173
6174             # This is the ONLY user of the PBS_ env vars set on top of this function!
6175             my $pbs_restore_cmd = [
6176                 '/usr/bin/pbs-restore',
6177                 '--repository', $repo,
6178                 $pbs_backup_name,
6179                 "$d->{devname}.img.fidx",
6180                 $path,
6181                 '--verbose',
6182                 ];
6183
6184             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6185             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6186
6187             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6188                 push @$pbs_restore_cmd, '--skip-zero';
6189             }
6190
6191             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6192             print "restore proxmox backup image: $dbg_cmdstring\n";
6193             run_command($pbs_restore_cmd);
6194         }
6195
6196         $fh->seek(0, 0) || die "seek failed - $!\n";
6197
6198         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6199
6200         my $cookie = { netcount => 0 };
6201         while (defined(my $line = <$fh>)) {
6202             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6203         }
6204
6205         $fh->close();
6206         $outfd->close();
6207     };
6208     my $err = $@;
6209
6210     $restore_deactivate_volumes->($storecfg, $devinfo);
6211
6212     rmtree $tmpdir;
6213
6214     if ($err) {
6215         unlink $tmpfn;
6216         $restore_destroy_volumes->($storecfg, $devinfo);
6217         die $err;
6218     }
6219
6220     rename($tmpfn, $conffile) ||
6221         die "unable to commit configuration file '$conffile'\n";
6222
6223     PVE::Cluster::cfs_update(); # make sure we read new file
6224
6225     eval { rescan($vmid, 1); };
6226     warn $@ if $@;
6227 }
6228
6229 sub restore_vma_archive {
6230     my ($archive, $vmid, $user, $opts, $comp) = @_;
6231
6232     my $readfrom = $archive;
6233
6234     my $cfg = PVE::Storage::config();
6235     my $commands = [];
6236     my $bwlimit = $opts->{bwlimit};
6237
6238     my $dbg_cmdstring = '';
6239     my $add_pipe = sub {
6240         my ($cmd) = @_;
6241         push @$commands, $cmd;
6242         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6243         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6244         $readfrom = '-';
6245     };
6246
6247     my $input = undef;
6248     if ($archive eq '-') {
6249         $input = '<&STDIN';
6250     } else {
6251         # If we use a backup from a PVE defined storage we also consider that
6252         # storage's rate limit:
6253         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6254         if (defined($volid)) {
6255             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6256             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6257             if ($readlimit) {
6258                 print STDERR "applying read rate limit: $readlimit\n";
6259                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6260                 $add_pipe->($cstream);
6261             }
6262         }
6263     }
6264
6265     if ($comp) {
6266         my $info = PVE::Storage::decompressor_info('vma', $comp);
6267         my $cmd = $info->{decompressor};
6268         push @$cmd, $readfrom;
6269         $add_pipe->($cmd);
6270     }
6271
6272     my $tmpdir = "/var/tmp/vzdumptmp$$";
6273     rmtree $tmpdir;
6274
6275     # disable interrupts (always do cleanups)
6276     local $SIG{INT} =
6277         local $SIG{TERM} =
6278         local $SIG{QUIT} =
6279         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6280
6281     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6282     POSIX::mkfifo($mapfifo, 0600);
6283     my $fifofh;
6284     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6285
6286     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6287
6288     my $oldtimeout;
6289     my $timeout = 5;
6290
6291     my $devinfo = {};
6292
6293     my $rpcenv = PVE::RPCEnvironment::get();
6294
6295     my $conffile = PVE::QemuConfig->config_file($vmid);
6296     my $tmpfn = "$conffile.$$.tmp";
6297
6298     # Note: $oldconf is undef if VM does not exist
6299     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6300     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6301
6302     my %storage_limits;
6303
6304     my $print_devmap = sub {
6305         my $cfgfn = "$tmpdir/qemu-server.conf";
6306
6307         # we can read the config - that is already extracted
6308         my $fh = IO::File->new($cfgfn, "r") ||
6309             die "unable to read qemu-server.conf - $!\n";
6310
6311         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6312         if (-f $fwcfgfn) {
6313             my $pve_firewall_dir = '/etc/pve/firewall';
6314             mkdir $pve_firewall_dir; # make sure the dir exists
6315             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6316         }
6317
6318         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6319
6320         foreach my $key (keys %storage_limits) {
6321             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6322             next if !$limit;
6323             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6324             $storage_limits{$key} = $limit * 1024;
6325         }
6326
6327         foreach my $devname (keys %$devinfo) {
6328             die "found no device mapping information for device '$devname'\n"
6329                 if !$devinfo->{$devname}->{virtdev};
6330         }
6331
6332         # create empty/temp config
6333         if ($oldconf) {
6334             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6335             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6336         }
6337
6338         # allocate volumes
6339         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6340
6341         # print restore information to $fifofh
6342         foreach my $virtdev (sort keys %$virtdev_hash) {
6343             my $d = $virtdev_hash->{$virtdev};
6344             next if $d->{is_cloudinit}; # no need to restore cloudinit
6345
6346             my $storeid = $d->{storeid};
6347             my $volid = $d->{volid};
6348
6349             my $map_opts = '';
6350             if (my $limit = $storage_limits{$storeid}) {
6351                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6352             }
6353
6354             my $write_zeros = 1;
6355             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6356                 $write_zeros = 0;
6357             }
6358
6359             my $path = PVE::Storage::path($cfg, $volid);
6360
6361             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6362
6363             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6364         }
6365
6366         $fh->seek(0, 0) || die "seek failed - $!\n";
6367
6368         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6369
6370         my $cookie = { netcount => 0 };
6371         while (defined(my $line = <$fh>)) {
6372             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6373         }
6374
6375         $fh->close();
6376         $outfd->close();
6377     };
6378
6379     eval {
6380         # enable interrupts
6381         local $SIG{INT} =
6382             local $SIG{TERM} =
6383             local $SIG{QUIT} =
6384             local $SIG{HUP} =
6385             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6386         local $SIG{ALRM} = sub { die "got timeout\n"; };
6387
6388         $oldtimeout = alarm($timeout);
6389
6390         my $parser = sub {
6391             my $line = shift;
6392
6393             print "$line\n";
6394
6395             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6396                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6397                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6398             } elsif ($line =~ m/^CTIME: /) {
6399                 # we correctly received the vma config, so we can disable
6400                 # the timeout now for disk allocation (set to 10 minutes, so
6401                 # that we always timeout if something goes wrong)
6402                 alarm(600);
6403                 &$print_devmap();
6404                 print $fifofh "done\n";
6405                 my $tmp = $oldtimeout || 0;
6406                 $oldtimeout = undef;
6407                 alarm($tmp);
6408                 close($fifofh);
6409                 $fifofh = undef;
6410             }
6411         };
6412
6413         print "restore vma archive: $dbg_cmdstring\n";
6414         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6415     };
6416     my $err = $@;
6417
6418     alarm($oldtimeout) if $oldtimeout;
6419
6420     $restore_deactivate_volumes->($cfg, $devinfo);
6421
6422     close($fifofh) if $fifofh;
6423     unlink $mapfifo;
6424     rmtree $tmpdir;
6425
6426     if ($err) {
6427         unlink $tmpfn;
6428         $restore_destroy_volumes->($cfg, $devinfo);
6429         die $err;
6430     }
6431
6432     rename($tmpfn, $conffile) ||
6433         die "unable to commit configuration file '$conffile'\n";
6434
6435     PVE::Cluster::cfs_update(); # make sure we read new file
6436
6437     eval { rescan($vmid, 1); };
6438     warn $@ if $@;
6439 }
6440
6441 sub restore_tar_archive {
6442     my ($archive, $vmid, $user, $opts) = @_;
6443
6444     if ($archive ne '-') {
6445         my $firstfile = tar_archive_read_firstfile($archive);
6446         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6447             if $firstfile ne 'qemu-server.conf';
6448     }
6449
6450     my $storecfg = PVE::Storage::config();
6451
6452     # avoid zombie disks when restoring over an existing VM -> cleanup first
6453     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6454     # skiplock=1 because qmrestore has set the 'create' lock itself already
6455     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6456     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6457
6458     my $tocmd = "/usr/lib/qemu-server/qmextract";
6459
6460     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6461     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6462     $tocmd .= ' --prealloc' if $opts->{prealloc};
6463     $tocmd .= ' --info' if $opts->{info};
6464
6465     # tar option "xf" does not autodetect compression when read from STDIN,
6466     # so we pipe to zcat
6467     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6468         PVE::Tools::shellquote("--to-command=$tocmd");
6469
6470     my $tmpdir = "/var/tmp/vzdumptmp$$";
6471     mkpath $tmpdir;
6472
6473     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6474     local $ENV{VZDUMP_VMID} = $vmid;
6475     local $ENV{VZDUMP_USER} = $user;
6476
6477     my $conffile = PVE::QemuConfig->config_file($vmid);
6478     my $tmpfn = "$conffile.$$.tmp";
6479
6480     # disable interrupts (always do cleanups)
6481     local $SIG{INT} =
6482         local $SIG{TERM} =
6483         local $SIG{QUIT} =
6484         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6485
6486     eval {
6487         # enable interrupts
6488         local $SIG{INT} =
6489             local $SIG{TERM} =
6490             local $SIG{QUIT} =
6491             local $SIG{HUP} =
6492             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6493
6494         if ($archive eq '-') {
6495             print "extracting archive from STDIN\n";
6496             run_command($cmd, input => "<&STDIN");
6497         } else {
6498             print "extracting archive '$archive'\n";
6499             run_command($cmd);
6500         }
6501
6502         return if $opts->{info};
6503
6504         # read new mapping
6505         my $map = {};
6506         my $statfile = "$tmpdir/qmrestore.stat";
6507         if (my $fd = IO::File->new($statfile, "r")) {
6508             while (defined (my $line = <$fd>)) {
6509                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6510                     $map->{$1} = $2 if $1;
6511                 } else {
6512                     print STDERR "unable to parse line in statfile - $line\n";
6513                 }
6514             }
6515             $fd->close();
6516         }
6517
6518         my $confsrc = "$tmpdir/qemu-server.conf";
6519
6520         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6521
6522         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6523
6524         my $cookie = { netcount => 0 };
6525         while (defined (my $line = <$srcfd>)) {
6526             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6527         }
6528
6529         $srcfd->close();
6530         $outfd->close();
6531     };
6532     if (my $err = $@) {
6533         unlink $tmpfn;
6534         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6535         die $err;
6536     }
6537
6538     rmtree $tmpdir;
6539
6540     rename $tmpfn, $conffile ||
6541         die "unable to commit configuration file '$conffile'\n";
6542
6543     PVE::Cluster::cfs_update(); # make sure we read new file
6544
6545     eval { rescan($vmid, 1); };
6546     warn $@ if $@;
6547 };
6548
6549 sub foreach_storage_used_by_vm {
6550     my ($conf, $func) = @_;
6551
6552     my $sidhash = {};
6553
6554     PVE::QemuConfig->foreach_volume($conf, sub {
6555         my ($ds, $drive) = @_;
6556         return if drive_is_cdrom($drive);
6557
6558         my $volid = $drive->{file};
6559
6560         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6561         $sidhash->{$sid} = $sid if $sid;
6562     });
6563
6564     foreach my $sid (sort keys %$sidhash) {
6565         &$func($sid);
6566     }
6567 }
6568
6569 my $qemu_snap_storage = {
6570     rbd => 1,
6571 };
6572 sub do_snapshots_with_qemu {
6573     my ($storecfg, $volid) = @_;
6574
6575     my $storage_name = PVE::Storage::parse_volume_id($volid);
6576     my $scfg = $storecfg->{ids}->{$storage_name};
6577     die "could not find storage '$storage_name'\n" if !defined($scfg);
6578
6579     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6580         return 1;
6581     }
6582
6583     if ($volid =~ m/\.(qcow2|qed)$/){
6584         return 1;
6585     }
6586
6587     return;
6588 }
6589
6590 sub qga_check_running {
6591     my ($vmid, $nowarn) = @_;
6592
6593     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6594     if ($@) {
6595         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6596         return 0;
6597     }
6598     return 1;
6599 }
6600
6601 sub template_create {
6602     my ($vmid, $conf, $disk) = @_;
6603
6604     my $storecfg = PVE::Storage::config();
6605
6606     PVE::QemuConfig->foreach_volume($conf, sub {
6607         my ($ds, $drive) = @_;
6608
6609         return if drive_is_cdrom($drive);
6610         return if $disk && $ds ne $disk;
6611
6612         my $volid = $drive->{file};
6613         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6614
6615         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6616         $drive->{file} = $voliddst;
6617         $conf->{$ds} = print_drive($drive);
6618         PVE::QemuConfig->write_config($vmid, $conf);
6619     });
6620 }
6621
6622 sub convert_iscsi_path {
6623     my ($path) = @_;
6624
6625     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6626         my $portal = $1;
6627         my $target = $2;
6628         my $lun = $3;
6629
6630         my $initiator_name = get_initiator_name();
6631
6632         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6633                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6634     }
6635
6636     die "cannot convert iscsi path '$path', unkown format\n";
6637 }
6638
6639 sub qemu_img_convert {
6640     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6641
6642     my $storecfg = PVE::Storage::config();
6643     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6644     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6645
6646     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6647
6648     my $cachemode;
6649     my $src_path;
6650     my $src_is_iscsi = 0;
6651     my $src_format;
6652
6653     if ($src_storeid) {
6654         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6655         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6656         $src_format = qemu_img_format($src_scfg, $src_volname);
6657         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6658         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6659         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6660     } elsif (-f $src_volid) {
6661         $src_path = $src_volid;
6662         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6663             $src_format = $1;
6664         }
6665     }
6666
6667     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6668
6669     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6670     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6671     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6672     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6673
6674     my $cmd = [];
6675     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6676     push @$cmd, '-l', "snapshot.name=$snapname"
6677         if $snapname && $src_format && $src_format eq "qcow2";
6678     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6679     push @$cmd, '-T', $cachemode if defined($cachemode);
6680
6681     if ($src_is_iscsi) {
6682         push @$cmd, '--image-opts';
6683         $src_path = convert_iscsi_path($src_path);
6684     } elsif ($src_format) {
6685         push @$cmd, '-f', $src_format;
6686     }
6687
6688     if ($dst_is_iscsi) {
6689         push @$cmd, '--target-image-opts';
6690         $dst_path = convert_iscsi_path($dst_path);
6691     } else {
6692         push @$cmd, '-O', $dst_format;
6693     }
6694
6695     push @$cmd, $src_path;
6696
6697     if (!$dst_is_iscsi && $is_zero_initialized) {
6698         push @$cmd, "zeroinit:$dst_path";
6699     } else {
6700         push @$cmd, $dst_path;
6701     }
6702
6703     my $parser = sub {
6704         my $line = shift;
6705         if($line =~ m/\((\S+)\/100\%\)/){
6706             my $percent = $1;
6707             my $transferred = int($size * $percent / 100);
6708             my $remaining = $size - $transferred;
6709
6710             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6711         }
6712
6713     };
6714
6715     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6716     my $err = $@;
6717     die "copy failed: $err" if $err;
6718 }
6719
6720 sub qemu_img_format {
6721     my ($scfg, $volname) = @_;
6722
6723     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6724         return $1;
6725     } else {
6726         return "raw";
6727     }
6728 }
6729
6730 sub qemu_drive_mirror {
6731     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6732
6733     $jobs = {} if !$jobs;
6734
6735     my $qemu_target;
6736     my $format;
6737     $jobs->{"drive-$drive"} = {};
6738
6739     if ($dst_volid =~ /^nbd:/) {
6740         $qemu_target = $dst_volid;
6741         $format = "nbd";
6742     } else {
6743         my $storecfg = PVE::Storage::config();
6744         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6745
6746         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6747
6748         $format = qemu_img_format($dst_scfg, $dst_volname);
6749
6750         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6751
6752         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6753     }
6754
6755     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6756     $opts->{format} = $format if $format;
6757
6758     if (defined($src_bitmap)) {
6759         $opts->{sync} = 'incremental';
6760         $opts->{bitmap} = $src_bitmap;
6761         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6762     }
6763
6764     if (defined($bwlimit)) {
6765         $opts->{speed} = $bwlimit * 1024;
6766         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6767     } else {
6768         print "drive mirror is starting for drive-$drive\n";
6769     }
6770
6771     # if a job already runs for this device we get an error, catch it for cleanup
6772     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6773     if (my $err = $@) {
6774         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6775         warn "$@\n" if $@;
6776         die "mirroring error: $err\n";
6777     }
6778
6779     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6780 }
6781
6782 # $completion can be either
6783 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6784 # 'cancel': wait until all jobs are ready, block-job-cancel them
6785 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6786 sub qemu_drive_mirror_monitor {
6787     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6788
6789     $completion //= 'complete';
6790
6791     eval {
6792         my $err_complete = 0;
6793
6794         while (1) {
6795             die "storage migration timed out\n" if $err_complete > 300;
6796
6797             my $stats = mon_cmd($vmid, "query-block-jobs");
6798
6799             my $running_mirror_jobs = {};
6800             foreach my $stat (@$stats) {
6801                 next if $stat->{type} ne 'mirror';
6802                 $running_mirror_jobs->{$stat->{device}} = $stat;
6803             }
6804
6805             my $readycounter = 0;
6806
6807             foreach my $job (keys %$jobs) {
6808
6809                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6810                     print "$job : finished\n";
6811                     delete $jobs->{$job};
6812                     next;
6813                 }
6814
6815                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6816
6817                 my $busy = $running_mirror_jobs->{$job}->{busy};
6818                 my $ready = $running_mirror_jobs->{$job}->{ready};
6819                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6820                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6821                     my $remaining = $total - $transferred;
6822                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6823
6824                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6825                 }
6826
6827                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6828             }
6829
6830             last if scalar(keys %$jobs) == 0;
6831
6832             if ($readycounter == scalar(keys %$jobs)) {
6833                 print "all mirroring jobs are ready \n";
6834                 last if $completion eq 'skip'; #do the complete later
6835
6836                 if ($vmiddst && $vmiddst != $vmid) {
6837                     my $agent_running = $qga && qga_check_running($vmid);
6838                     if ($agent_running) {
6839                         print "freeze filesystem\n";
6840                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6841                     } else {
6842                         print "suspend vm\n";
6843                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6844                     }
6845
6846                     # if we clone a disk for a new target vm, we don't switch the disk
6847                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6848
6849                     if ($agent_running) {
6850                         print "unfreeze filesystem\n";
6851                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6852                     } else {
6853                         print "resume vm\n";
6854                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6855                     }
6856
6857                     last;
6858                 } else {
6859
6860                     foreach my $job (keys %$jobs) {
6861                         # try to switch the disk if source and destination are on the same guest
6862                         print "$job: Completing block job...\n";
6863
6864                         my $op;
6865                         if ($completion eq 'complete') {
6866                             $op = 'block-job-complete';
6867                         } elsif ($completion eq 'cancel') {
6868                             $op = 'block-job-cancel';
6869                         } else {
6870                             die "invalid completion value: $completion\n";
6871                         }
6872                         eval { mon_cmd($vmid, $op, device => $job) };
6873                         if ($@ =~ m/cannot be completed/) {
6874                             print "$job: Block job cannot be completed, try again.\n";
6875                             $err_complete++;
6876                         }else {
6877                             print "$job: Completed successfully.\n";
6878                             $jobs->{$job}->{complete} = 1;
6879                         }
6880                     }
6881                 }
6882             }
6883             sleep 1;
6884         }
6885     };
6886     my $err = $@;
6887
6888     if ($err) {
6889         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6890         die "mirroring error: $err";
6891     }
6892
6893 }
6894
6895 sub qemu_blockjobs_cancel {
6896     my ($vmid, $jobs) = @_;
6897
6898     foreach my $job (keys %$jobs) {
6899         print "$job: Cancelling block job\n";
6900         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6901         $jobs->{$job}->{cancel} = 1;
6902     }
6903
6904     while (1) {
6905         my $stats = mon_cmd($vmid, "query-block-jobs");
6906
6907         my $running_jobs = {};
6908         foreach my $stat (@$stats) {
6909             $running_jobs->{$stat->{device}} = $stat;
6910         }
6911
6912         foreach my $job (keys %$jobs) {
6913
6914             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6915                 print "$job: Done.\n";
6916                 delete $jobs->{$job};
6917             }
6918         }
6919
6920         last if scalar(keys %$jobs) == 0;
6921
6922         sleep 1;
6923     }
6924 }
6925
6926 sub clone_disk {
6927     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6928         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6929
6930     my $newvolid;
6931
6932     if (!$full) {
6933         print "create linked clone of drive $drivename ($drive->{file})\n";
6934         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6935         push @$newvollist, $newvolid;
6936     } else {
6937
6938         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6939         $storeid = $storage if $storage;
6940
6941         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6942
6943         print "create full clone of drive $drivename ($drive->{file})\n";
6944         my $name = undef;
6945         my $size = undef;
6946         if (drive_is_cloudinit($drive)) {
6947             $name = "vm-$newvmid-cloudinit";
6948             $name .= ".$dst_format" if $dst_format ne 'raw';
6949             $snapname = undef;
6950             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6951         } elsif ($drivename eq 'efidisk0') {
6952             $size = get_efivars_size($conf);
6953         } else {
6954             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
6955         }
6956         $size /= 1024;
6957         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
6958         push @$newvollist, $newvolid;
6959
6960         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6961
6962         if (drive_is_cloudinit($drive)) {
6963             goto no_data_clone;
6964         }
6965
6966         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6967         if (!$running || $snapname) {
6968             # TODO: handle bwlimits
6969             if ($drivename eq 'efidisk0') {
6970                 # the relevant data on the efidisk may be smaller than the source
6971                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6972                 # that is given by the OVMF_VARS.fd
6973                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6974                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6975                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6976                     "if=$src_path", "of=$dst_path"]);
6977             } else {
6978                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6979             }
6980         } else {
6981
6982             my $kvmver = get_running_qemu_version ($vmid);
6983             if (!min_version($kvmver, 2, 7)) {
6984                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6985                     if $drive->{iothread};
6986             }
6987
6988             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6989                 $completion, $qga, $bwlimit);
6990         }
6991     }
6992
6993 no_data_clone:
6994     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
6995
6996     my $disk = $drive;
6997     $disk->{format} = undef;
6998     $disk->{file} = $newvolid;
6999     $disk->{size} = $size if defined($size);
7000
7001     return $disk;
7002 }
7003
7004 sub get_running_qemu_version {
7005     my ($vmid) = @_;
7006     my $res = mon_cmd($vmid, "query-version");
7007     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7008 }
7009
7010 sub qemu_use_old_bios_files {
7011     my ($machine_type) = @_;
7012
7013     return if !$machine_type;
7014
7015     my $use_old_bios_files = undef;
7016
7017     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7018         $machine_type = $1;
7019         $use_old_bios_files = 1;
7020     } else {
7021         my $version = extract_version($machine_type, kvm_user_version());
7022         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7023         # load new efi bios files on migration. So this hack is required to allow
7024         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7025         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7026         $use_old_bios_files = !min_version($version, 2, 4);
7027     }
7028
7029     return ($use_old_bios_files, $machine_type);
7030 }
7031
7032 sub get_efivars_size {
7033     my ($conf) = @_;
7034     my $arch = get_vm_arch($conf);
7035     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7036     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7037     return -s $ovmf_vars;
7038 }
7039
7040 sub update_efidisk_size {
7041     my ($conf) = @_;
7042
7043     return if !defined($conf->{efidisk0});
7044
7045     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7046     $disk->{size} = get_efivars_size($conf);
7047     $conf->{efidisk0} = print_drive($disk);
7048
7049     return;
7050 }
7051
7052 sub create_efidisk($$$$$) {
7053     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7054
7055     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7056     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7057
7058     my $vars_size_b = -s $ovmf_vars;
7059     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7060     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7061     PVE::Storage::activate_volumes($storecfg, [$volid]);
7062
7063     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7064     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7065
7066     return ($volid, $size/1024);
7067 }
7068
7069 sub vm_iothreads_list {
7070     my ($vmid) = @_;
7071
7072     my $res = mon_cmd($vmid, 'query-iothreads');
7073
7074     my $iothreads = {};
7075     foreach my $iothread (@$res) {
7076         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7077     }
7078
7079     return $iothreads;
7080 }
7081
7082 sub scsihw_infos {
7083     my ($conf, $drive) = @_;
7084
7085     my $maxdev = 0;
7086
7087     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7088         $maxdev = 7;
7089     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7090         $maxdev = 1;
7091     } else {
7092         $maxdev = 256;
7093     }
7094
7095     my $controller = int($drive->{index} / $maxdev);
7096     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7097         ? "virtioscsi"
7098         : "scsihw";
7099
7100     return ($maxdev, $controller, $controller_prefix);
7101 }
7102
7103 sub windows_version {
7104     my ($ostype) = @_;
7105
7106     return 0 if !$ostype;
7107
7108     my $winversion = 0;
7109
7110     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7111         $winversion = 5;
7112     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7113         $winversion = 6;
7114     } elsif ($ostype =~ m/^win(\d+)$/) {
7115         $winversion = $1;
7116     }
7117
7118     return $winversion;
7119 }
7120
7121 sub resolve_dst_disk_format {
7122         my ($storecfg, $storeid, $src_volname, $format) = @_;
7123         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7124
7125         if (!$format) {
7126             # if no target format is specified, use the source disk format as hint
7127             if ($src_volname) {
7128                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7129                 $format = qemu_img_format($scfg, $src_volname);
7130             } else {
7131                 return $defFormat;
7132             }
7133         }
7134
7135         # test if requested format is supported - else use default
7136         my $supported = grep { $_ eq $format } @$validFormats;
7137         $format = $defFormat if !$supported;
7138         return $format;
7139 }
7140
7141 # NOTE: if this logic changes, please update docs & possibly gui logic
7142 sub find_vmstate_storage {
7143     my ($conf, $storecfg) = @_;
7144
7145     # first, return storage from conf if set
7146     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7147
7148     my ($target, $shared, $local);
7149
7150     foreach_storage_used_by_vm($conf, sub {
7151         my ($sid) = @_;
7152         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7153         my $dst = $scfg->{shared} ? \$shared : \$local;
7154         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7155     });
7156
7157     # second, use shared storage where VM has at least one disk
7158     # third, use local storage where VM has at least one disk
7159     # fall back to local storage
7160     $target = $shared // $local // 'local';
7161
7162     return $target;
7163 }
7164
7165 sub generate_uuid {
7166     my ($uuid, $uuid_str);
7167     UUID::generate($uuid);
7168     UUID::unparse($uuid, $uuid_str);
7169     return $uuid_str;
7170 }
7171
7172 sub generate_smbios1_uuid {
7173     return "uuid=".generate_uuid();
7174 }
7175
7176 sub nbd_stop {
7177     my ($vmid) = @_;
7178
7179     mon_cmd($vmid, 'nbd-server-stop');
7180 }
7181
7182 sub create_reboot_request {
7183     my ($vmid) = @_;
7184     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7185         or die "failed to create reboot trigger file: $!\n";
7186     close($fh);
7187 }
7188
7189 sub clear_reboot_request {
7190     my ($vmid) = @_;
7191     my $path = "/run/qemu-server/$vmid.reboot";
7192     my $res = 0;
7193
7194     $res = unlink($path);
7195     die "could not remove reboot request for $vmid: $!"
7196         if !$res && $! != POSIX::ENOENT;
7197
7198     return $res;
7199 }
7200
7201 sub bootorder_from_legacy {
7202     my ($conf, $bootcfg) = @_;
7203
7204     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7205     my $bootindex_hash = {};
7206     my $i = 1;
7207     foreach my $o (split(//, $boot)) {
7208         $bootindex_hash->{$o} = $i*100;
7209         $i++;
7210     }
7211
7212     my $bootorder = {};
7213
7214     PVE::QemuConfig->foreach_volume($conf, sub {
7215         my ($ds, $drive) = @_;
7216
7217         if (drive_is_cdrom ($drive, 1)) {
7218             if ($bootindex_hash->{d}) {
7219                 $bootorder->{$ds} = $bootindex_hash->{d};
7220                 $bootindex_hash->{d} += 1;
7221             }
7222         } elsif ($bootindex_hash->{c}) {
7223             $bootorder->{$ds} = $bootindex_hash->{c}
7224                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7225             $bootindex_hash->{c} += 1;
7226         }
7227     });
7228
7229     if ($bootindex_hash->{n}) {
7230         for (my $i = 0; $i < $MAX_NETS; $i++) {
7231             my $netname = "net$i";
7232             next if !$conf->{$netname};
7233             $bootorder->{$netname} = $bootindex_hash->{n};
7234             $bootindex_hash->{n} += 1;
7235         }
7236     }
7237
7238     return $bootorder;
7239 }
7240
7241 # Generate default device list for 'boot: order=' property. Matches legacy
7242 # default boot order, but with explicit device names. This is important, since
7243 # the fallback for when neither 'order' nor the old format is specified relies
7244 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7245 sub get_default_bootdevices {
7246     my ($conf) = @_;
7247
7248     my @ret = ();
7249
7250     # harddisk
7251     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7252     push @ret, $first if $first;
7253
7254     # cdrom
7255     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7256     push @ret, $first if $first;
7257
7258     # network
7259     for (my $i = 0; $i < $MAX_NETS; $i++) {
7260         my $netname = "net$i";
7261         next if !$conf->{$netname};
7262         push @ret, $netname;
7263         last;
7264     }
7265
7266     return \@ret;
7267 }
7268
7269 sub device_bootorder {
7270     my ($conf) = @_;
7271
7272     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7273
7274     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7275
7276     my $bootorder = {};
7277     if (!defined($boot) || $boot->{legacy}) {
7278         $bootorder = bootorder_from_legacy($conf, $boot);
7279     } elsif ($boot->{order}) {
7280         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7281         for my $dev (PVE::Tools::split_list($boot->{order})) {
7282             $bootorder->{$dev} = $i++;
7283         }
7284     }
7285
7286     return $bootorder;
7287 }
7288
7289 # bash completion helper
7290
7291 sub complete_backup_archives {
7292     my ($cmdname, $pname, $cvalue) = @_;
7293
7294     my $cfg = PVE::Storage::config();
7295
7296     my $storeid;
7297
7298     if ($cvalue =~ m/^([^:]+):/) {
7299         $storeid = $1;
7300     }
7301
7302     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7303
7304     my $res = [];
7305     foreach my $id (keys %$data) {
7306         foreach my $item (@{$data->{$id}}) {
7307             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7308             push @$res, $item->{volid} if defined($item->{volid});
7309         }
7310     }
7311
7312     return $res;
7313 }
7314
7315 my $complete_vmid_full = sub {
7316     my ($running) = @_;
7317
7318     my $idlist = vmstatus();
7319
7320     my $res = [];
7321
7322     foreach my $id (keys %$idlist) {
7323         my $d = $idlist->{$id};
7324         if (defined($running)) {
7325             next if $d->{template};
7326             next if $running && $d->{status} ne 'running';
7327             next if !$running && $d->{status} eq 'running';
7328         }
7329         push @$res, $id;
7330
7331     }
7332     return $res;
7333 };
7334
7335 sub complete_vmid {
7336     return &$complete_vmid_full();
7337 }
7338
7339 sub complete_vmid_stopped {
7340     return &$complete_vmid_full(0);
7341 }
7342
7343 sub complete_vmid_running {
7344     return &$complete_vmid_full(1);
7345 }
7346
7347 sub complete_storage {
7348
7349     my $cfg = PVE::Storage::config();
7350     my $ids = $cfg->{ids};
7351
7352     my $res = [];
7353     foreach my $sid (keys %$ids) {
7354         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7355         next if !$ids->{$sid}->{content}->{images};
7356         push @$res, $sid;
7357     }
7358
7359     return $res;
7360 }
7361
7362 sub complete_migration_storage {
7363     my ($cmd, $param, $current_value, $all_args) = @_;
7364
7365     my $targetnode = @$all_args[1];
7366
7367     my $cfg = PVE::Storage::config();
7368     my $ids = $cfg->{ids};
7369
7370     my $res = [];
7371     foreach my $sid (keys %$ids) {
7372         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7373         next if !$ids->{$sid}->{content}->{images};
7374         push @$res, $sid;
7375     }
7376
7377     return $res;
7378 }
7379
7380 1;