PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     keephugepages => {
 445         optional => 1,
 446         type => 'boolean',
 447         default => 0,
 448         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 449             ." after VM shutdown and can be used for subsequent starts.",
 450     },
 451     vcpus => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "Number of hotplugged vcpus.",
 455         minimum => 1,
 456         default => 0,
 457     },
 458     acpi => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable ACPI.",
 462         default => 1,
 463     },
 464     agent => {
 465         optional => 1,
 466         description => "Enable/disable Qemu GuestAgent and its properties.",
 467         type => 'string',
 468         format => $agent_fmt,
 469     },
 470     kvm => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable KVM hardware virtualization.",
 474         default => 1,
 475     },
 476     tdf => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable time drift fix.",
 480         default => 0,
 481     },
 482     localtime => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Set the real time clock to local time. This is enabled by default if ostype"
 486             ." indicates a Microsoft OS.",
 487     },
 488     freeze => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 492     },
 493     vga => {
 494         optional => 1,
 495         type => 'string', format => $vga_fmt,
 496         description => "Configure the VGA hardware.",
 497         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 498             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 499             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 500             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 501             ." display server. For win* OS you can select how many independent displays you want,"
 502             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 503             ." using a serial device as terminal.",
 504     },
 505     watchdog => {
 506         optional => 1,
 507         type => 'string', format => 'pve-qm-watchdog',
 508         description => "Create a virtual hardware watchdog device.",
 509         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 510             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 511             ." else the watchdog will reset the guest (or execute the respective action specified)",
 512     },
 513     startdate => {
 514         optional => 1,
 515         type => 'string',
 516         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 517         description => "Set the initial date of the real time clock. Valid format for date are:"
 518             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 519         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 520         default => 'now',
 521     },
 522     startup =>  get_standard_option('pve-startup-order'),
 523     template => {
 524         optional => 1,
 525         type => 'boolean',
 526         description => "Enable/disable Template.",
 527         default => 0,
 528     },
 529     args => {
 530         optional => 1,
 531         type => 'string',
 532         description => "Arbitrary arguments passed to kvm.",
 533         verbose_description => <<EODESCR,
 534 Arbitrary arguments passed to kvm, for example:
 535
 536 args: -no-reboot -no-hpet
 537
 538 NOTE: this option is for experts only.
 539 EODESCR
 540     },
 541     tablet => {
 542         optional => 1,
 543         type => 'boolean',
 544         default => 1,
 545         description => "Enable/disable the USB tablet device.",
 546         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 547             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 548             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 549             ." may consider disabling this to save some context switches. This is turned off by"
 550             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 551     },
 552     migrate_speed => {
 553         optional => 1,
 554         type => 'integer',
 555         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 556         minimum => 0,
 557         default => 0,
 558     },
 559     migrate_downtime => {
 560         optional => 1,
 561         type => 'number',
 562         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 563         minimum => 0,
 564         default => 0.1,
 565     },
 566     cdrom => {
 567         optional => 1,
 568         type => 'string', format => 'pve-qm-ide',
 569         typetext => '<volume>',
 570         description => "This is an alias for option -ide2",
 571     },
 572     cpu => {
 573         optional => 1,
 574         description => "Emulated CPU type.",
 575         type => 'string',
 576         format => 'pve-vm-cpu-conf',
 577     },
 578     parent => get_standard_option('pve-snapshot-name', {
 579         optional => 1,
 580         description => "Parent snapshot name. This is used internally, and should not be modified.",
 581     }),
 582     snaptime => {
 583         optional => 1,
 584         description => "Timestamp for snapshots.",
 585         type => 'integer',
 586         minimum => 0,
 587     },
 588     vmstate => {
 589         optional => 1,
 590         type => 'string', format => 'pve-volume-id',
 591         description => "Reference to a volume which stores the VM state. This is used internally"
 592             ." for snapshots.",
 593     },
 594     vmstatestorage => get_standard_option('pve-storage-id', {
 595         description => "Default storage for VM state volumes/files.",
 596         optional => 1,
 597     }),
 598     runningmachine => get_standard_option('pve-qemu-machine', {
 599         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 600             ." for snapshots.",
 601     }),
 602     runningcpu => {
 603         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 604             ." internally for snapshots.",
 605         optional => 1,
 606         type => 'string',
 607         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 608         format_description => 'QEMU -cpu parameter'
 609     },
 610     machine => get_standard_option('pve-qemu-machine'),
 611     arch => {
 612         description => "Virtual processor architecture. Defaults to the host.",
 613         optional => 1,
 614         type => 'string',
 615         enum => [qw(x86_64 aarch64)],
 616     },
 617     smbios1 => {
 618         description => "Specify SMBIOS type 1 fields.",
 619         type => 'string', format => 'pve-qm-smbios1',
 620         maxLength => 512,
 621         optional => 1,
 622     },
 623     protection => {
 624         optional => 1,
 625         type => 'boolean',
 626         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 627             ." remove disk operations.",
 628         default => 0,
 629     },
 630     bios => {
 631         optional => 1,
 632         type => 'string',
 633         enum => [ qw(seabios ovmf) ],
 634         description => "Select BIOS implementation.",
 635         default => 'seabios',
 636     },
 637     vmgenid => {
 638         type => 'string',
 639         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 640         format_description => 'UUID',
 641         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 642             ." to disable explicitly.",
 643         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 644             ." value identifier to the guest OS. This allows to notify the guest operating system"
 645             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 646             ." execution or creation from a template). The guest operating system notices the"
 647             ." change, and is then able to react as appropriate by marking its copies of"
 648             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 649             ."Note that auto-creation only works when done through API/CLI create or update methods"
 650             .", but not when manually editing the config file.",
 651         default => "1 (autogenerated)",
 652         optional => 1,
 653     },
 654     hookscript => {
 655         type => 'string',
 656         format => 'pve-volume-id',
 657         optional => 1,
 658         description => "Script that will be executed during various steps in the vms lifetime.",
 659     },
 660     ivshmem => {
 661         type => 'string',
 662         format => $ivshmem_fmt,
 663         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 664             ." the host.",
 665         optional => 1,
 666     },
 667     audio0 => {
 668         type => 'string',
 669         format => $audio_fmt,
 670         description => "Configure a audio device, useful in combination with QXL/Spice.",
 671         optional => 1
 672     },
 673     spice_enhancements => {
 674         type => 'string',
 675         format => $spice_enhancements_fmt,
 676         description => "Configure additional enhancements for SPICE.",
 677         optional => 1
 678     },
 679     tags => {
 680         type => 'string', format => 'pve-tag-list',
 681         description => 'Tags of the VM. This is only meta information.',
 682         optional => 1,
 683     },
 684     rng0 => {
 685         type => 'string',
 686         format => $rng_fmt,
 687         description => "Configure a VirtIO-based Random Number Generator.",
 688         optional => 1,
 689     },
 690 };
 691
 692 my $cicustom_fmt = {
 693     meta => {
 694         type => 'string',
 695         optional => 1,
 696         description => 'Specify a custom file containing all meta data passed to the VM via"
 697             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 698         format => 'pve-volume-id',
 699         format_description => 'volume',
 700     },
 701     network => {
 702         type => 'string',
 703         optional => 1,
 704         description => 'Specify a custom file containing all network data passed to the VM via'
 705             .' cloud-init.',
 706         format => 'pve-volume-id',
 707         format_description => 'volume',
 708     },
 709     user => {
 710         type => 'string',
 711         optional => 1,
 712         description => 'Specify a custom file containing all user data passed to the VM via'
 713             .' cloud-init.',
 714         format => 'pve-volume-id',
 715         format_description => 'volume',
 716     },
 717 };
 718 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 719
 720 my $confdesc_cloudinit = {
 721     citype => {
 722         optional => 1,
 723         type => 'string',
 724         description => 'Specifies the cloud-init configuration format. The default depends on the'
 725             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 726             .' and `configdrive2` for windows.',
 727         enum => ['configdrive2', 'nocloud'],
 728     },
 729     ciuser => {
 730         optional => 1,
 731         type => 'string',
 732         description => "cloud-init: User name to change ssh keys and password for instead of the"
 733             ." image's configured default user.",
 734     },
 735     cipassword => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'cloud-init: Password to assign the user. Using this is generally not'
 739             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 740             .' support hashed passwords.',
 741     },
 742     cicustom => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Specify custom files to replace the automatically generated'
 746             .' ones at start.',
 747         format => 'pve-qm-cicustom',
 748     },
 749     searchdomain => {
 750         optional => 1,
 751         type => 'string',
 752         description => "cloud-init: Sets DNS search domains for a container. Create will'
 753             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 754             .' are set.",
 755     },
 756     nameserver => {
 757         optional => 1,
 758         type => 'string', format => 'address-list',
 759         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 760             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 761             .' are set.",
 762     },
 763     sshkeys => {
 764         optional => 1,
 765         type => 'string',
 766         format => 'urlencoded',
 767         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 768     },
 769 };
 770
 771 # what about other qemu settings ?
 772 #cpu => 'string',
 773 #machine => 'string',
 774 #fda => 'file',
 775 #fdb => 'file',
 776 #mtdblock => 'file',
 777 #sd => 'file',
 778 #pflash => 'file',
 779 #snapshot => 'bool',
 780 #bootp => 'file',
 781 ##tftp => 'dir',
 782 ##smb => 'dir',
 783 #kernel => 'file',
 784 #append => 'string',
 785 #initrd => 'file',
 786 ##soundhw => 'string',
 787
 788 while (my ($k, $v) = each %$confdesc) {
 789     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 790 }
 791
 792 my $MAX_USB_DEVICES = 5;
 793 my $MAX_NETS = 32;
 794 my $MAX_SERIAL_PORTS = 4;
 795 my $MAX_PARALLEL_PORTS = 3;
 796 my $MAX_NUMA = 8;
 797
 798 my $numa_fmt = {
 799     cpus => {
 800         type => "string",
 801         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 802         description => "CPUs accessing this NUMA node.",
 803         format_description => "id[-id];...",
 804     },
 805     memory => {
 806         type => "number",
 807         description => "Amount of memory this NUMA node provides.",
 808         optional => 1,
 809     },
 810     hostnodes => {
 811         type => "string",
 812         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 813         description => "Host NUMA nodes to use.",
 814         format_description => "id[-id];...",
 815         optional => 1,
 816     },
 817     policy => {
 818         type => 'string',
 819         enum => [qw(preferred bind interleave)],
 820         description => "NUMA allocation policy.",
 821         optional => 1,
 822     },
 823 };
 824 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 825 my $numadesc = {
 826     optional => 1,
 827     type => 'string', format => $numa_fmt,
 828     description => "NUMA topology.",
 829 };
 830 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 831
 832 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 833     $confdesc->{"numa$i"} = $numadesc;
 834 }
 835
 836 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 837                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 838                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 839 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 840
 841 my $net_fmt_bridge_descr = <<__EOD__;
 842 Bridge to attach the network device to. The Proxmox VE standard bridge
 843 is called 'vmbr0'.
 844
 845 If you do not specify a bridge, we create a kvm user (NATed) network
 846 device, which provides DHCP and DNS services. The following addresses
 847 are used:
 848
 849  10.0.2.2   Gateway
 850  10.0.2.3   DNS Server
 851  10.0.2.4   SMB Server
 852
 853 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 854 __EOD__
 855
 856 my $net_fmt = {
 857     macaddr  => get_standard_option('mac-addr', {
 858         description => "MAC address. That address must be unique withing your network. This is"
 859             ." automatically generated if not specified.",
 860     }),
 861     model => {
 862         type => 'string',
 863         description => "Network Card Model. The 'virtio' model provides the best performance with"
 864             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 865             ." best to use 'e1000'.",
 866         enum => $nic_model_list,
 867         default_key => 1,
 868     },
 869     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 870     bridge => {
 871         type => 'string',
 872         description => $net_fmt_bridge_descr,
 873         format_description => 'bridge',
 874         pattern => '[-_.\w\d]+',
 875         optional => 1,
 876     },
 877     queues => {
 878         type => 'integer',
 879         minimum => 0, maximum => 16,
 880         description => 'Number of packet queues to be used on the device.',
 881         optional => 1,
 882     },
 883     rate => {
 884         type => 'number',
 885         minimum => 0,
 886         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 887         optional => 1,
 888     },
 889     tag => {
 890         type => 'integer',
 891         minimum => 1, maximum => 4094,
 892         description => 'VLAN tag to apply to packets on this interface.',
 893         optional => 1,
 894     },
 895     trunks => {
 896         type => 'string',
 897         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 898         description => 'VLAN trunks to pass through this interface.',
 899         format_description => 'vlanid[;vlanid...]',
 900         optional => 1,
 901     },
 902     firewall => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be protected by the firewall.',
 905         optional => 1,
 906     },
 907     link_down => {
 908         type => 'boolean',
 909         description => 'Whether this interface should be disconnected (like pulling the plug).',
 910         optional => 1,
 911     },
 912     mtu => {
 913         type => 'integer',
 914         minimum => 1, maximum => 65520,
 915         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 916         optional => 1,
 917     },
 918 };
 919
 920 my $netdesc = {
 921     optional => 1,
 922     type => 'string', format => $net_fmt,
 923     description => "Specify network devices.",
 924 };
 925
 926 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 927
 928 my $ipconfig_fmt = {
 929     ip => {
 930         type => 'string',
 931         format => 'pve-ipv4-config',
 932         format_description => 'IPv4Format/CIDR',
 933         description => 'IPv4 address in CIDR format.',
 934         optional => 1,
 935         default => 'dhcp',
 936     },
 937     gw => {
 938         type => 'string',
 939         format => 'ipv4',
 940         format_description => 'GatewayIPv4',
 941         description => 'Default gateway for IPv4 traffic.',
 942         optional => 1,
 943         requires => 'ip',
 944     },
 945     ip6 => {
 946         type => 'string',
 947         format => 'pve-ipv6-config',
 948         format_description => 'IPv6Format/CIDR',
 949         description => 'IPv6 address in CIDR format.',
 950         optional => 1,
 951         default => 'dhcp',
 952     },
 953     gw6 => {
 954         type => 'string',
 955         format => 'ipv6',
 956         format_description => 'GatewayIPv6',
 957         description => 'Default gateway for IPv6 traffic.',
 958         optional => 1,
 959         requires => 'ip6',
 960     },
 961 };
 962 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 963 my $ipconfigdesc = {
 964     optional => 1,
 965     type => 'string', format => 'pve-qm-ipconfig',
 966     description => <<'EODESCR',
 967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 968
 969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 970
 971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 972 gateway should be provided.
 973 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 974
 975 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 976 dhcp on IPv4.
 977 EODESCR
 978 };
 979 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 980
 981 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 982     $confdesc->{"net$i"} = $netdesc;
 983     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 984 }
 985
 986 foreach my $key (keys %$confdesc_cloudinit) {
 987     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 988 }
 989
 990 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 991 sub verify_volume_id_or_qm_path {
 992     my ($volid, $noerr) = @_;
 993
 994     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 995         return $volid;
 996     }
 997
 998     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 999     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1000     if ($@) {
1001         return undef if $noerr;
1002         die $@;
1003     }
1004     return $volid;
1005 }
1006
1007 my $usb_fmt = {
1008     host => {
1009         default_key => 1,
1010         type => 'string', format => 'pve-qm-usb-device',
1011         format_description => 'HOSTUSBDEVICE|spice',
1012         description => <<EODESCR,
1013 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1014
1015  'bus-port(.port)*' (decimal numbers) or
1016  'vendor_id:product_id' (hexadeciaml numbers) or
1017  'spice'
1018
1019 You can use the 'lsusb -t' command to list existing usb devices.
1020
1021 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1022 machines - use with special care.
1023
1024 The value 'spice' can be used to add a usb redirection devices for spice.
1025 EODESCR
1026     },
1027     usb3 => {
1028         optional => 1,
1029         type => 'boolean',
1030         description => "Specifies whether if given host option is a USB3 device or port.",
1031         default => 0,
1032     },
1033 };
1034
1035 my $usbdesc = {
1036     optional => 1,
1037     type => 'string', format => $usb_fmt,
1038     description => "Configure an USB device (n is 0 to 4).",
1039 };
1040 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1041
1042 my $serialdesc = {
1043         optional => 1,
1044         type => 'string',
1045         pattern => '(/dev/.+|socket)',
1046         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1047         verbose_description =>  <<EODESCR,
1048 Create a serial device inside the VM (n is 0 to 3), and pass through a
1049 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1050 host side (use 'qm terminal' to open a terminal connection).
1051
1052 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1053 use with special care.
1054
1055 CAUTION: Experimental! User reported problems with this option.
1056 EODESCR
1057 };
1058
1059 my $paralleldesc= {
1060         optional => 1,
1061         type => 'string',
1062         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1063         description =>  "Map host parallel devices (n is 0 to 2).",
1064         verbose_description =>  <<EODESCR,
1065 Map host parallel devices (n is 0 to 2).
1066
1067 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1068 machines - use with special care.
1069
1070 CAUTION: Experimental! User reported problems with this option.
1071 EODESCR
1072 };
1073
1074 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1075     $confdesc->{"parallel$i"} = $paralleldesc;
1076 }
1077
1078 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1079     $confdesc->{"serial$i"} = $serialdesc;
1080 }
1081
1082 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1083     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1084 }
1085
1086 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1087     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1088 }
1089
1090 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1091     $confdesc->{"usb$i"} = $usbdesc;
1092 }
1093
1094 my $kvm_api_version = 0;
1095
1096 sub kvm_version {
1097     return $kvm_api_version if $kvm_api_version;
1098
1099     open my $fh, '<', '/dev/kvm'
1100         or return undef;
1101
1102     # 0xae00 => KVM_GET_API_VERSION
1103     $kvm_api_version = ioctl($fh, 0xae00, 0);
1104
1105     return $kvm_api_version;
1106 }
1107
1108 my $kvm_user_version = {};
1109 my $kvm_mtime = {};
1110
1111 sub kvm_user_version {
1112     my ($binary) = @_;
1113
1114     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1115     my $st = stat($binary);
1116
1117     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1118     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1119         $cachedmtime == $st->mtime;
1120
1121     $kvm_user_version->{$binary} = 'unknown';
1122     $kvm_mtime->{$binary} = $st->mtime;
1123
1124     my $code = sub {
1125         my $line = shift;
1126         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1127             $kvm_user_version->{$binary} = $2;
1128         }
1129     };
1130
1131     eval { run_command([$binary, '--version'], outfunc => $code); };
1132     warn $@ if $@;
1133
1134     return $kvm_user_version->{$binary};
1135
1136 }
1137 my sub extract_version {
1138     my ($machine_type, $version) = @_;
1139     $version = kvm_user_version() if !defined($version);
1140     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1141 }
1142
1143 sub kernel_has_vhost_net {
1144     return -c '/dev/vhost-net';
1145 }
1146
1147 sub option_exists {
1148     my $key = shift;
1149     return defined($confdesc->{$key});
1150 }
1151
1152 my $cdrom_path;
1153 sub get_cdrom_path {
1154
1155     return  $cdrom_path if $cdrom_path;
1156
1157     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1158     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1159     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1160 }
1161
1162 sub get_iso_path {
1163     my ($storecfg, $vmid, $cdrom) = @_;
1164
1165     if ($cdrom eq 'cdrom') {
1166         return get_cdrom_path();
1167     } elsif ($cdrom eq 'none') {
1168         return '';
1169     } elsif ($cdrom =~ m|^/|) {
1170         return $cdrom;
1171     } else {
1172         return PVE::Storage::path($storecfg, $cdrom);
1173     }
1174 }
1175
1176 # try to convert old style file names to volume IDs
1177 sub filename_to_volume_id {
1178     my ($vmid, $file, $media) = @_;
1179
1180      if (!($file eq 'none' || $file eq 'cdrom' ||
1181           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1182
1183         return undef if $file =~ m|/|;
1184
1185         if ($media && $media eq 'cdrom') {
1186             $file = "local:iso/$file";
1187         } else {
1188             $file = "local:$vmid/$file";
1189         }
1190     }
1191
1192     return $file;
1193 }
1194
1195 sub verify_media_type {
1196     my ($opt, $vtype, $media) = @_;
1197
1198     return if !$media;
1199
1200     my $etype;
1201     if ($media eq 'disk') {
1202         $etype = 'images';
1203     } elsif ($media eq 'cdrom') {
1204         $etype = 'iso';
1205     } else {
1206         die "internal error";
1207     }
1208
1209     return if ($vtype eq $etype);
1210
1211     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1212 }
1213
1214 sub cleanup_drive_path {
1215     my ($opt, $storecfg, $drive) = @_;
1216
1217     # try to convert filesystem paths to volume IDs
1218
1219     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1220         ($drive->{file} !~ m|^/dev/.+|) &&
1221         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1222         ($drive->{file} !~ m/^\d+$/)) {
1223         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1224         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1225             if !$vtype;
1226         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1227         verify_media_type($opt, $vtype, $drive->{media});
1228         $drive->{file} = $volid;
1229     }
1230
1231     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1232 }
1233
1234 sub parse_hotplug_features {
1235     my ($data) = @_;
1236
1237     my $res = {};
1238
1239     return $res if $data eq '0';
1240
1241     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1242
1243     foreach my $feature (PVE::Tools::split_list($data)) {
1244         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1245             $res->{$1} = 1;
1246         } else {
1247             die "invalid hotplug feature '$feature'\n";
1248         }
1249     }
1250     return $res;
1251 }
1252
1253 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1254 sub pve_verify_hotplug_features {
1255     my ($value, $noerr) = @_;
1256
1257     return $value if parse_hotplug_features($value);
1258
1259     return undef if $noerr;
1260
1261     die "unable to parse hotplug option\n";
1262 }
1263
1264 sub scsi_inquiry {
1265     my($fh, $noerr) = @_;
1266
1267     my $SG_IO = 0x2285;
1268     my $SG_GET_VERSION_NUM = 0x2282;
1269
1270     my $versionbuf = "\x00" x 8;
1271     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1272     if (!$ret) {
1273         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1274         return undef;
1275     }
1276     my $version = unpack("I", $versionbuf);
1277     if ($version < 30000) {
1278         die "scsi generic interface too old\n"  if !$noerr;
1279         return undef;
1280     }
1281
1282     my $buf = "\x00" x 36;
1283     my $sensebuf = "\x00" x 8;
1284     my $cmd = pack("C x3 C x1", 0x12, 36);
1285
1286     # see /usr/include/scsi/sg.h
1287     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1288
1289     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1290                       length($sensebuf), 0, length($buf), $buf,
1291                       $cmd, $sensebuf, 6000);
1292
1293     $ret = ioctl($fh, $SG_IO, $packet);
1294     if (!$ret) {
1295         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1296         return undef;
1297     }
1298
1299     my @res = unpack($sg_io_hdr_t, $packet);
1300     if ($res[17] || $res[18]) {
1301         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1302         return undef;
1303     }
1304
1305     my $res = {};
1306     (my $byte0, my $byte1, $res->{vendor},
1307      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1308
1309     $res->{removable} = $byte1 & 128 ? 1 : 0;
1310     $res->{type} = $byte0 & 31;
1311
1312     return $res;
1313 }
1314
1315 sub path_is_scsi {
1316     my ($path) = @_;
1317
1318     my $fh = IO::File->new("+<$path") || return undef;
1319     my $res = scsi_inquiry($fh, 1);
1320     close($fh);
1321
1322     return $res;
1323 }
1324
1325 sub print_tabletdevice_full {
1326     my ($conf, $arch) = @_;
1327
1328     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1329
1330     # we use uhci for old VMs because tablet driver was buggy in older qemu
1331     my $usbbus;
1332     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1333         $usbbus = 'ehci';
1334     } else {
1335         $usbbus = 'uhci';
1336     }
1337
1338     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1339 }
1340
1341 sub print_keyboarddevice_full {
1342     my ($conf, $arch, $machine) = @_;
1343
1344     return undef if $arch ne 'aarch64';
1345
1346     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1347 }
1348
1349 sub print_drivedevice_full {
1350     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1351
1352     my $device = '';
1353     my $maxdev = 0;
1354
1355     my $drive_id = "$drive->{interface}$drive->{index}";
1356     if ($drive->{interface} eq 'virtio') {
1357         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1358         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1359         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1360     } elsif ($drive->{interface} eq 'scsi') {
1361
1362         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1363         my $unit = $drive->{index} % $maxdev;
1364         my $devicetype = 'hd';
1365         my $path = '';
1366         if (drive_is_cdrom($drive)) {
1367             $devicetype = 'cd';
1368         } else {
1369             if ($drive->{file} =~ m|^/|) {
1370                 $path = $drive->{file};
1371                 if (my $info = path_is_scsi($path)) {
1372                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1373                         $devicetype = 'block';
1374                     } elsif ($info->{type} == 1) { # tape
1375                         $devicetype = 'generic';
1376                     }
1377                 }
1378             } else {
1379                  $path = PVE::Storage::path($storecfg, $drive->{file});
1380             }
1381
1382             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1383             my $version = extract_version($machine_type, kvm_user_version());
1384             if ($path =~ m/^iscsi\:\/\// &&
1385                !min_version($version, 4, 1)) {
1386                 $devicetype = 'generic';
1387             }
1388         }
1389
1390         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1391             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1392         } else {
1393             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1394                 .",lun=$drive->{index}";
1395         }
1396         $device .= ",drive=drive-$drive_id,id=$drive_id";
1397
1398         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1399             $device .= ",rotation_rate=1";
1400         }
1401         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1402
1403     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1404         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1405         my $controller = int($drive->{index} / $maxdev);
1406         my $unit = $drive->{index} % $maxdev;
1407         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1408
1409         $device = "ide-$devicetype";
1410         if ($drive->{interface} eq 'ide') {
1411             $device .= ",bus=ide.$controller,unit=$unit";
1412         } else {
1413             $device .= ",bus=ahci$controller.$unit";
1414         }
1415         $device .= ",drive=drive-$drive_id,id=$drive_id";
1416
1417         if ($devicetype eq 'hd') {
1418             if (my $model = $drive->{model}) {
1419                 $model = URI::Escape::uri_unescape($model);
1420                 $device .= ",model=$model";
1421             }
1422             if ($drive->{ssd}) {
1423                 $device .= ",rotation_rate=1";
1424             }
1425         }
1426         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1427     } elsif ($drive->{interface} eq 'usb') {
1428         die "implement me";
1429         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1430     } else {
1431         die "unsupported interface type";
1432     }
1433
1434     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1435
1436     if (my $serial = $drive->{serial}) {
1437         $serial = URI::Escape::uri_unescape($serial);
1438         $device .= ",serial=$serial";
1439     }
1440
1441
1442     return $device;
1443 }
1444
1445 sub get_initiator_name {
1446     my $initiator;
1447
1448     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1449     while (defined(my $line = <$fh>)) {
1450         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1451         $initiator = $1;
1452         last;
1453     }
1454     $fh->close();
1455
1456     return $initiator;
1457 }
1458
1459 sub print_drive_commandline_full {
1460     my ($storecfg, $vmid, $drive) = @_;
1461
1462     my $path;
1463     my $volid = $drive->{file};
1464     my $format;
1465
1466     if (drive_is_cdrom($drive)) {
1467         $path = get_iso_path($storecfg, $vmid, $volid);
1468     } else {
1469         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1470         if ($storeid) {
1471             $path = PVE::Storage::path($storecfg, $volid);
1472             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1473             $format = qemu_img_format($scfg, $volname);
1474         } else {
1475             $path = $volid;
1476             $format = "raw";
1477         }
1478    }
1479
1480     my $opts = '';
1481     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1482     foreach my $o (@qemu_drive_options) {
1483         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1484     }
1485
1486     # snapshot only accepts on|off
1487     if (defined($drive->{snapshot})) {
1488         my $v = $drive->{snapshot} ? 'on' : 'off';
1489         $opts .= ",snapshot=$v";
1490     }
1491
1492     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1493         my ($dir, $qmpname) = @$type;
1494         if (my $v = $drive->{"mbps$dir"}) {
1495             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1496         }
1497         if (my $v = $drive->{"mbps${dir}_max"}) {
1498             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1499         }
1500         if (my $v = $drive->{"bps${dir}_max_length"}) {
1501             $opts .= ",throttling.bps$qmpname-max-length=$v";
1502         }
1503         if (my $v = $drive->{"iops${dir}"}) {
1504             $opts .= ",throttling.iops$qmpname=$v";
1505         }
1506         if (my $v = $drive->{"iops${dir}_max"}) {
1507             $opts .= ",throttling.iops$qmpname-max=$v";
1508         }
1509         if (my $v = $drive->{"iops${dir}_max_length"}) {
1510             $opts .= ",throttling.iops$qmpname-max-length=$v";
1511         }
1512     }
1513
1514     $opts .= ",format=$format" if $format && !$drive->{format};
1515
1516     my $cache_direct = 0;
1517
1518     if (my $cache = $drive->{cache}) {
1519         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1520     } elsif (!drive_is_cdrom($drive)) {
1521         $opts .= ",cache=none";
1522         $cache_direct = 1;
1523     }
1524
1525     # aio native works only with O_DIRECT
1526     if (!$drive->{aio}) {
1527         if($cache_direct) {
1528             $opts .= ",aio=native";
1529         } else {
1530             $opts .= ",aio=threads";
1531         }
1532     }
1533
1534     if (!drive_is_cdrom($drive)) {
1535         my $detectzeroes;
1536         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1537             $detectzeroes = 'off';
1538         } elsif ($drive->{discard}) {
1539             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1540         } else {
1541             # This used to be our default with discard not being specified:
1542             $detectzeroes = 'on';
1543         }
1544         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1545     }
1546
1547     my $pathinfo = $path ? "file=$path," : '';
1548
1549     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1550 }
1551
1552 sub print_netdevice_full {
1553     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1554
1555     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1556
1557     my $device = $net->{model};
1558     if ($net->{model} eq 'virtio') {
1559          $device = 'virtio-net-pci';
1560      };
1561
1562     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1563     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1564     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1565         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1566         # and out of each queue plus one config interrupt and control vector queue
1567         my $vectors = $net->{queues} * 2 + 2;
1568         $tmpstr .= ",vectors=$vectors,mq=on";
1569     }
1570     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1571
1572     if (my $mtu = $net->{mtu}) {
1573         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1574             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1575             if ($mtu == 1) {
1576                  $mtu = $bridge_mtu;
1577             } elsif ($mtu < 576) {
1578                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1579             } elsif ($mtu > $bridge_mtu) {
1580                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1581             }
1582             $tmpstr .= ",host_mtu=$mtu";
1583         } else {
1584             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1585         }
1586     }
1587
1588     if ($use_old_bios_files) {
1589         my $romfile;
1590         if ($device eq 'virtio-net-pci') {
1591             $romfile = 'pxe-virtio.rom';
1592         } elsif ($device eq 'e1000') {
1593             $romfile = 'pxe-e1000.rom';
1594         } elsif ($device eq 'ne2k') {
1595             $romfile = 'pxe-ne2k_pci.rom';
1596         } elsif ($device eq 'pcnet') {
1597             $romfile = 'pxe-pcnet.rom';
1598         } elsif ($device eq 'rtl8139') {
1599             $romfile = 'pxe-rtl8139.rom';
1600         }
1601         $tmpstr .= ",romfile=$romfile" if $romfile;
1602     }
1603
1604     return $tmpstr;
1605 }
1606
1607 sub print_netdev_full {
1608     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1609
1610     my $i = '';
1611     if ($netid =~ m/^net(\d+)$/) {
1612         $i = int($1);
1613     }
1614
1615     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1616
1617     my $ifname = "tap${vmid}i$i";
1618
1619     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1620     die "interface name '$ifname' is too long (max 15 character)\n"
1621         if length($ifname) >= 16;
1622
1623     my $vhostparam = '';
1624     if (is_native($arch)) {
1625         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1626     }
1627
1628     my $vmname = $conf->{name} || "vm$vmid";
1629
1630     my $netdev = "";
1631     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1632
1633     if ($net->{bridge}) {
1634         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1635             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1636     } else {
1637         $netdev = "type=user,id=$netid,hostname=$vmname";
1638     }
1639
1640     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1641
1642     return $netdev;
1643 }
1644
1645 my $vga_map = {
1646     'cirrus' => 'cirrus-vga',
1647     'std' => 'VGA',
1648     'vmware' => 'vmware-svga',
1649     'virtio' => 'virtio-vga',
1650 };
1651
1652 sub print_vga_device {
1653     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1654
1655     my $type = $vga_map->{$vga->{type}};
1656     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1657         $type = 'virtio-gpu';
1658     }
1659     my $vgamem_mb = $vga->{memory};
1660
1661     my $max_outputs = '';
1662     if ($qxlnum) {
1663         $type = $id ? 'qxl' : 'qxl-vga';
1664
1665         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1666             # set max outputs so linux can have up to 4 qxl displays with one device
1667             if (min_version($machine_version, 4, 1)) {
1668                 $max_outputs = ",max_outputs=4";
1669             }
1670         }
1671     }
1672
1673     die "no devicetype for $vga->{type}\n" if !$type;
1674
1675     my $memory = "";
1676     if ($vgamem_mb) {
1677         if ($vga->{type} eq 'virtio') {
1678             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1679             $memory = ",max_hostmem=$bytes";
1680         } elsif ($qxlnum) {
1681             # from https://www.spice-space.org/multiple-monitors.html
1682             $memory = ",vgamem_mb=$vga->{memory}";
1683             my $ram = $vgamem_mb * 4;
1684             my $vram = $vgamem_mb * 2;
1685             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1686         } else {
1687             $memory = ",vgamem_mb=$vga->{memory}";
1688         }
1689     } elsif ($qxlnum && $id) {
1690         $memory = ",ram_size=67108864,vram_size=33554432";
1691     }
1692
1693     my $edidoff = "";
1694     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1695         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1696     }
1697
1698     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1699     my $vgaid = "vga" . ($id // '');
1700     my $pciaddr;
1701
1702     if ($q35 && $vgaid eq 'vga') {
1703         # the first display uses pcie.0 bus on q35 machines
1704         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1705     } else {
1706         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1707     }
1708
1709     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1710 }
1711
1712 sub parse_number_sets {
1713     my ($set) = @_;
1714     my $res = [];
1715     foreach my $part (split(/;/, $set)) {
1716         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1717             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1718             push @$res, [ $1, $2 ];
1719         } else {
1720             die "invalid range: $part\n";
1721         }
1722     }
1723     return $res;
1724 }
1725
1726 sub parse_numa {
1727     my ($data) = @_;
1728
1729     my $res = parse_property_string($numa_fmt, $data);
1730     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1731     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1732     return $res;
1733 }
1734
1735 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1736 sub parse_net {
1737     my ($data) = @_;
1738
1739     my $res = eval { parse_property_string($net_fmt, $data) };
1740     if ($@) {
1741         warn $@;
1742         return undef;
1743     }
1744     if (!defined($res->{macaddr})) {
1745         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1746         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1747     }
1748     return $res;
1749 }
1750
1751 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1752 sub parse_ipconfig {
1753     my ($data) = @_;
1754
1755     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1756     if ($@) {
1757         warn $@;
1758         return undef;
1759     }
1760
1761     if ($res->{gw} && !$res->{ip}) {
1762         warn 'gateway specified without specifying an IP address';
1763         return undef;
1764     }
1765     if ($res->{gw6} && !$res->{ip6}) {
1766         warn 'IPv6 gateway specified without specifying an IPv6 address';
1767         return undef;
1768     }
1769     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1770         warn 'gateway specified together with DHCP';
1771         return undef;
1772     }
1773     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1774         # gw6 + auto/dhcp
1775         warn "IPv6 gateway specified together with $res->{ip6} address";
1776         return undef;
1777     }
1778
1779     if (!$res->{ip} && !$res->{ip6}) {
1780         return { ip => 'dhcp', ip6 => 'dhcp' };
1781     }
1782
1783     return $res;
1784 }
1785
1786 sub print_net {
1787     my $net = shift;
1788
1789     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1790 }
1791
1792 sub add_random_macs {
1793     my ($settings) = @_;
1794
1795     foreach my $opt (keys %$settings) {
1796         next if $opt !~ m/^net(\d+)$/;
1797         my $net = parse_net($settings->{$opt});
1798         next if !$net;
1799         $settings->{$opt} = print_net($net);
1800     }
1801 }
1802
1803 sub vm_is_volid_owner {
1804     my ($storecfg, $vmid, $volid) = @_;
1805
1806     if ($volid !~  m|^/|) {
1807         my ($path, $owner);
1808         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1809         if ($owner && ($owner == $vmid)) {
1810             return 1;
1811         }
1812     }
1813
1814     return undef;
1815 }
1816
1817 sub vmconfig_register_unused_drive {
1818     my ($storecfg, $vmid, $conf, $drive) = @_;
1819
1820     if (drive_is_cloudinit($drive)) {
1821         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1822         warn $@ if $@;
1823     } elsif (!drive_is_cdrom($drive)) {
1824         my $volid = $drive->{file};
1825         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1826             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1827         }
1828     }
1829 }
1830
1831 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1832 my $smbios1_fmt = {
1833     uuid => {
1834         type => 'string',
1835         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1836         format_description => 'UUID',
1837         description => "Set SMBIOS1 UUID.",
1838         optional => 1,
1839     },
1840     version => {
1841         type => 'string',
1842         pattern => '[A-Za-z0-9+\/]+={0,2}',
1843         format_description => 'Base64 encoded string',
1844         description => "Set SMBIOS1 version.",
1845         optional => 1,
1846     },
1847     serial => {
1848         type => 'string',
1849         pattern => '[A-Za-z0-9+\/]+={0,2}',
1850         format_description => 'Base64 encoded string',
1851         description => "Set SMBIOS1 serial number.",
1852         optional => 1,
1853     },
1854     manufacturer => {
1855         type => 'string',
1856         pattern => '[A-Za-z0-9+\/]+={0,2}',
1857         format_description => 'Base64 encoded string',
1858         description => "Set SMBIOS1 manufacturer.",
1859         optional => 1,
1860     },
1861     product => {
1862         type => 'string',
1863         pattern => '[A-Za-z0-9+\/]+={0,2}',
1864         format_description => 'Base64 encoded string',
1865         description => "Set SMBIOS1 product ID.",
1866         optional => 1,
1867     },
1868     sku => {
1869         type => 'string',
1870         pattern => '[A-Za-z0-9+\/]+={0,2}',
1871         format_description => 'Base64 encoded string',
1872         description => "Set SMBIOS1 SKU string.",
1873         optional => 1,
1874     },
1875     family => {
1876         type => 'string',
1877         pattern => '[A-Za-z0-9+\/]+={0,2}',
1878         format_description => 'Base64 encoded string',
1879         description => "Set SMBIOS1 family string.",
1880         optional => 1,
1881     },
1882     base64 => {
1883         type => 'boolean',
1884         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1885         optional => 1,
1886     },
1887 };
1888
1889 sub parse_smbios1 {
1890     my ($data) = @_;
1891
1892     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1893     warn $@ if $@;
1894     return $res;
1895 }
1896
1897 sub print_smbios1 {
1898     my ($smbios1) = @_;
1899     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1900 }
1901
1902 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1903
1904 sub parse_watchdog {
1905     my ($value) = @_;
1906
1907     return undef if !$value;
1908
1909     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1910     warn $@ if $@;
1911     return $res;
1912 }
1913
1914 sub parse_guest_agent {
1915     my ($value) = @_;
1916
1917     return {} if !defined($value->{agent});
1918
1919     my $res = eval { parse_property_string($agent_fmt, $value->{agent}) };
1920     warn $@ if $@;
1921
1922     # if the agent is disabled ignore the other potentially set properties
1923     return {} if !$res->{enabled};
1924     return $res;
1925 }
1926
1927 sub parse_vga {
1928     my ($value) = @_;
1929
1930     return {} if !$value;
1931     my $res = eval { parse_property_string($vga_fmt, $value) };
1932     warn $@ if $@;
1933     return $res;
1934 }
1935
1936 sub parse_rng {
1937     my ($value) = @_;
1938
1939     return undef if !$value;
1940
1941     my $res = eval { parse_property_string($rng_fmt, $value) };
1942     warn $@ if $@;
1943     return $res;
1944 }
1945
1946 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1947 sub verify_usb_device {
1948     my ($value, $noerr) = @_;
1949
1950     return $value if parse_usb_device($value);
1951
1952     return undef if $noerr;
1953
1954     die "unable to parse usb device\n";
1955 }
1956
1957 # add JSON properties for create and set function
1958 sub json_config_properties {
1959     my $prop = shift;
1960
1961     foreach my $opt (keys %$confdesc) {
1962         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1963             $opt eq 'runningmachine' || $opt eq 'runningcpu';
1964         $prop->{$opt} = $confdesc->{$opt};
1965     }
1966
1967     return $prop;
1968 }
1969
1970 # return copy of $confdesc_cloudinit to generate documentation
1971 sub cloudinit_config_properties {
1972
1973     return dclone($confdesc_cloudinit);
1974 }
1975
1976 sub check_type {
1977     my ($key, $value) = @_;
1978
1979     die "unknown setting '$key'\n" if !$confdesc->{$key};
1980
1981     my $type = $confdesc->{$key}->{type};
1982
1983     if (!defined($value)) {
1984         die "got undefined value\n";
1985     }
1986
1987     if ($value =~ m/[\n\r]/) {
1988         die "property contains a line feed\n";
1989     }
1990
1991     if ($type eq 'boolean') {
1992         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1993         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1994         die "type check ('boolean') failed - got '$value'\n";
1995     } elsif ($type eq 'integer') {
1996         return int($1) if $value =~ m/^(\d+)$/;
1997         die "type check ('integer') failed - got '$value'\n";
1998     } elsif ($type eq 'number') {
1999         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2000         die "type check ('number') failed - got '$value'\n";
2001     } elsif ($type eq 'string') {
2002         if (my $fmt = $confdesc->{$key}->{format}) {
2003             PVE::JSONSchema::check_format($fmt, $value);
2004             return $value;
2005         }
2006         $value =~ s/^\"(.*)\"$/$1/;
2007         return $value;
2008     } else {
2009         die "internal error"
2010     }
2011 }
2012
2013 sub destroy_vm {
2014     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2015
2016     my $conf = PVE::QemuConfig->load_config($vmid);
2017
2018     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2019
2020     if ($conf->{template}) {
2021         # check if any base image is still used by a linked clone
2022         PVE::QemuConfig->foreach_volume($conf, sub {
2023                 my ($ds, $drive) = @_;
2024                 return if drive_is_cdrom($drive);
2025
2026                 my $volid = $drive->{file};
2027                 return if !$volid || $volid =~ m|^/|;
2028
2029                 die "base volume '$volid' is still in use by linked cloned\n"
2030                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2031
2032         });
2033     }
2034
2035     # only remove disks owned by this VM
2036     PVE::QemuConfig->foreach_volume($conf, sub {
2037         my ($ds, $drive) = @_;
2038         return if drive_is_cdrom($drive, 1);
2039
2040         my $volid = $drive->{file};
2041         return if !$volid || $volid =~ m|^/|;
2042
2043         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2044         return if !$path || !$owner || ($owner != $vmid);
2045
2046         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2047         warn "Could not remove disk '$volid', check manually: $@" if $@;
2048     });
2049
2050     # also remove unused disk
2051     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2052     PVE::Storage::foreach_volid($vmdisks, sub {
2053         my ($volid, $sid, $volname, $d) = @_;
2054         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2055         warn $@ if $@;
2056     });
2057
2058     if (defined $replacement_conf) {
2059         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2060     } else {
2061         PVE::QemuConfig->destroy_config($vmid);
2062     }
2063 }
2064
2065 sub parse_vm_config {
2066     my ($filename, $raw) = @_;
2067
2068     return undef if !defined($raw);
2069
2070     my $res = {
2071         digest => Digest::SHA::sha1_hex($raw),
2072         snapshots => {},
2073         pending => {},
2074     };
2075
2076     $filename =~ m|/qemu-server/(\d+)\.conf$|
2077         || die "got strange filename '$filename'";
2078
2079     my $vmid = $1;
2080
2081     my $conf = $res;
2082     my $descr;
2083     my $section = '';
2084
2085     my @lines = split(/\n/, $raw);
2086     foreach my $line (@lines) {
2087         next if $line =~ m/^\s*$/;
2088
2089         if ($line =~ m/^\[PENDING\]\s*$/i) {
2090             $section = 'pending';
2091             if (defined($descr)) {
2092                 $descr =~ s/\s+$//;
2093                 $conf->{description} = $descr;
2094             }
2095             $descr = undef;
2096             $conf = $res->{$section} = {};
2097             next;
2098
2099         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2100             $section = $1;
2101             if (defined($descr)) {
2102                 $descr =~ s/\s+$//;
2103                 $conf->{description} = $descr;
2104             }
2105             $descr = undef;
2106             $conf = $res->{snapshots}->{$section} = {};
2107             next;
2108         }
2109
2110         if ($line =~ m/^\#(.*)\s*$/) {
2111             $descr = '' if !defined($descr);
2112             $descr .= PVE::Tools::decode_text($1) . "\n";
2113             next;
2114         }
2115
2116         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2117             $descr = '' if !defined($descr);
2118             $descr .= PVE::Tools::decode_text($2);
2119         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2120             $conf->{snapstate} = $1;
2121         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2122             my $key = $1;
2123             my $value = $2;
2124             $conf->{$key} = $value;
2125         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2126             my $value = $1;
2127             if ($section eq 'pending') {
2128                 $conf->{delete} = $value; # we parse this later
2129             } else {
2130                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2131             }
2132         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2133             my $key = $1;
2134             my $value = $2;
2135             eval { $value = check_type($key, $value); };
2136             if ($@) {
2137                 warn "vm $vmid - unable to parse value of '$key' - $@";
2138             } else {
2139                 $key = 'ide2' if $key eq 'cdrom';
2140                 my $fmt = $confdesc->{$key}->{format};
2141                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2142                     my $v = parse_drive($key, $value);
2143                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2144                         $v->{file} = $volid;
2145                         $value = print_drive($v);
2146                     } else {
2147                         warn "vm $vmid - unable to parse value of '$key'\n";
2148                         next;
2149                     }
2150                 }
2151
2152                 $conf->{$key} = $value;
2153             }
2154         }
2155     }
2156
2157     if (defined($descr)) {
2158         $descr =~ s/\s+$//;
2159         $conf->{description} = $descr;
2160     }
2161     delete $res->{snapstate}; # just to be sure
2162
2163     return $res;
2164 }
2165
2166 sub write_vm_config {
2167     my ($filename, $conf) = @_;
2168
2169     delete $conf->{snapstate}; # just to be sure
2170
2171     if ($conf->{cdrom}) {
2172         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2173         $conf->{ide2} = $conf->{cdrom};
2174         delete $conf->{cdrom};
2175     }
2176
2177     # we do not use 'smp' any longer
2178     if ($conf->{sockets}) {
2179         delete $conf->{smp};
2180     } elsif ($conf->{smp}) {
2181         $conf->{sockets} = $conf->{smp};
2182         delete $conf->{cores};
2183         delete $conf->{smp};
2184     }
2185
2186     my $used_volids = {};
2187
2188     my $cleanup_config = sub {
2189         my ($cref, $pending, $snapname) = @_;
2190
2191         foreach my $key (keys %$cref) {
2192             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2193                 $key eq 'snapstate' || $key eq 'pending';
2194             my $value = $cref->{$key};
2195             if ($key eq 'delete') {
2196                 die "propertry 'delete' is only allowed in [PENDING]\n"
2197                     if !$pending;
2198                 # fixme: check syntax?
2199                 next;
2200             }
2201             eval { $value = check_type($key, $value); };
2202             die "unable to parse value of '$key' - $@" if $@;
2203
2204             $cref->{$key} = $value;
2205
2206             if (!$snapname && is_valid_drivename($key)) {
2207                 my $drive = parse_drive($key, $value);
2208                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2209             }
2210         }
2211     };
2212
2213     &$cleanup_config($conf);
2214
2215     &$cleanup_config($conf->{pending}, 1);
2216
2217     foreach my $snapname (keys %{$conf->{snapshots}}) {
2218         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2219         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2220     }
2221
2222     # remove 'unusedX' settings if we re-add a volume
2223     foreach my $key (keys %$conf) {
2224         my $value = $conf->{$key};
2225         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2226             delete $conf->{$key};
2227         }
2228     }
2229
2230     my $generate_raw_config = sub {
2231         my ($conf, $pending) = @_;
2232
2233         my $raw = '';
2234
2235         # add description as comment to top of file
2236         if (defined(my $descr = $conf->{description})) {
2237             if ($descr) {
2238                 foreach my $cl (split(/\n/, $descr)) {
2239                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2240                 }
2241             } else {
2242                 $raw .= "#\n" if $pending;
2243             }
2244         }
2245
2246         foreach my $key (sort keys %$conf) {
2247             next if $key =~ /^(digest|description|pending|snapshots)$/;
2248             $raw .= "$key: $conf->{$key}\n";
2249         }
2250         return $raw;
2251     };
2252
2253     my $raw = &$generate_raw_config($conf);
2254
2255     if (scalar(keys %{$conf->{pending}})){
2256         $raw .= "\n[PENDING]\n";
2257         $raw .= &$generate_raw_config($conf->{pending}, 1);
2258     }
2259
2260     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2261         $raw .= "\n[$snapname]\n";
2262         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2263     }
2264
2265     return $raw;
2266 }
2267
2268 sub load_defaults {
2269
2270     my $res = {};
2271
2272     # we use static defaults from our JSON schema configuration
2273     foreach my $key (keys %$confdesc) {
2274         if (defined(my $default = $confdesc->{$key}->{default})) {
2275             $res->{$key} = $default;
2276         }
2277     }
2278
2279     return $res;
2280 }
2281
2282 sub config_list {
2283     my $vmlist = PVE::Cluster::get_vmlist();
2284     my $res = {};
2285     return $res if !$vmlist || !$vmlist->{ids};
2286     my $ids = $vmlist->{ids};
2287     my $nodename = nodename();
2288
2289     foreach my $vmid (keys %$ids) {
2290         my $d = $ids->{$vmid};
2291         next if !$d->{node} || $d->{node} ne $nodename;
2292         next if !$d->{type} || $d->{type} ne 'qemu';
2293         $res->{$vmid}->{exists} = 1;
2294     }
2295     return $res;
2296 }
2297
2298 # test if VM uses local resources (to prevent migration)
2299 sub check_local_resources {
2300     my ($conf, $noerr) = @_;
2301
2302     my @loc_res = ();
2303
2304     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2305     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2306
2307     push @loc_res, "ivshmem" if $conf->{ivshmem};
2308
2309     foreach my $k (keys %$conf) {
2310         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2311         # sockets are safe: they will recreated be on the target side post-migrate
2312         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2313         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2314     }
2315
2316     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2317
2318     return \@loc_res;
2319 }
2320
2321 # check if used storages are available on all nodes (use by migrate)
2322 sub check_storage_availability {
2323     my ($storecfg, $conf, $node) = @_;
2324
2325     PVE::QemuConfig->foreach_volume($conf, sub {
2326         my ($ds, $drive) = @_;
2327
2328         my $volid = $drive->{file};
2329         return if !$volid;
2330
2331         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2332         return if !$sid;
2333
2334         # check if storage is available on both nodes
2335         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2336         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2337    });
2338 }
2339
2340 # list nodes where all VM images are available (used by has_feature API)
2341 sub shared_nodes {
2342     my ($conf, $storecfg) = @_;
2343
2344     my $nodelist = PVE::Cluster::get_nodelist();
2345     my $nodehash = { map { $_ => 1 } @$nodelist };
2346     my $nodename = nodename();
2347
2348     PVE::QemuConfig->foreach_volume($conf, sub {
2349         my ($ds, $drive) = @_;
2350
2351         my $volid = $drive->{file};
2352         return if !$volid;
2353
2354         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2355         if ($storeid) {
2356             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2357             if ($scfg->{disable}) {
2358                 $nodehash = {};
2359             } elsif (my $avail = $scfg->{nodes}) {
2360                 foreach my $node (keys %$nodehash) {
2361                     delete $nodehash->{$node} if !$avail->{$node};
2362                 }
2363             } elsif (!$scfg->{shared}) {
2364                 foreach my $node (keys %$nodehash) {
2365                     delete $nodehash->{$node} if $node ne $nodename
2366                 }
2367             }
2368         }
2369     });
2370
2371     return $nodehash
2372 }
2373
2374 sub check_local_storage_availability {
2375     my ($conf, $storecfg) = @_;
2376
2377     my $nodelist = PVE::Cluster::get_nodelist();
2378     my $nodehash = { map { $_ => {} } @$nodelist };
2379
2380     PVE::QemuConfig->foreach_volume($conf, sub {
2381         my ($ds, $drive) = @_;
2382
2383         my $volid = $drive->{file};
2384         return if !$volid;
2385
2386         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2387         if ($storeid) {
2388             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2389
2390             if ($scfg->{disable}) {
2391                 foreach my $node (keys %$nodehash) {
2392                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2393                 }
2394             } elsif (my $avail = $scfg->{nodes}) {
2395                 foreach my $node (keys %$nodehash) {
2396                     if (!$avail->{$node}) {
2397                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2398                     }
2399                 }
2400             }
2401         }
2402     });
2403
2404     foreach my $node (values %$nodehash) {
2405         if (my $unavail = $node->{unavailable_storages}) {
2406             $node->{unavailable_storages} = [ sort keys %$unavail ];
2407         }
2408     }
2409
2410     return $nodehash
2411 }
2412
2413 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2414 sub check_running {
2415     my ($vmid, $nocheck, $node) = @_;
2416
2417     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2418     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2419 }
2420
2421 sub vzlist {
2422
2423     my $vzlist = config_list();
2424
2425     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2426
2427     while (defined(my $de = $fd->read)) {
2428         next if $de !~ m/^(\d+)\.pid$/;
2429         my $vmid = $1;
2430         next if !defined($vzlist->{$vmid});
2431         if (my $pid = check_running($vmid)) {
2432             $vzlist->{$vmid}->{pid} = $pid;
2433         }
2434     }
2435
2436     return $vzlist;
2437 }
2438
2439 our $vmstatus_return_properties = {
2440     vmid => get_standard_option('pve-vmid'),
2441     status => {
2442         description => "Qemu process status.",
2443         type => 'string',
2444         enum => ['stopped', 'running'],
2445     },
2446     maxmem => {
2447         description => "Maximum memory in bytes.",
2448         type => 'integer',
2449         optional => 1,
2450         renderer => 'bytes',
2451     },
2452     maxdisk => {
2453         description => "Root disk size in bytes.",
2454         type => 'integer',
2455         optional => 1,
2456         renderer => 'bytes',
2457     },
2458     name => {
2459         description => "VM name.",
2460         type => 'string',
2461         optional => 1,
2462     },
2463     qmpstatus => {
2464         description => "Qemu QMP agent status.",
2465         type => 'string',
2466         optional => 1,
2467     },
2468     pid => {
2469         description => "PID of running qemu process.",
2470         type => 'integer',
2471         optional => 1,
2472     },
2473     uptime => {
2474         description => "Uptime.",
2475         type => 'integer',
2476         optional => 1,
2477         renderer => 'duration',
2478     },
2479     cpus => {
2480         description => "Maximum usable CPUs.",
2481         type => 'number',
2482         optional => 1,
2483     },
2484     lock => {
2485         description => "The current config lock, if any.",
2486         type => 'string',
2487         optional => 1,
2488     },
2489     tags => {
2490         description  => "The current configured tags, if any",
2491         type => 'string',
2492         optional => 1,
2493     },
2494 };
2495
2496 my $last_proc_pid_stat;
2497
2498 # get VM status information
2499 # This must be fast and should not block ($full == false)
2500 # We only query KVM using QMP if $full == true (this can be slow)
2501 sub vmstatus {
2502     my ($opt_vmid, $full) = @_;
2503
2504     my $res = {};
2505
2506     my $storecfg = PVE::Storage::config();
2507
2508     my $list = vzlist();
2509     my $defaults = load_defaults();
2510
2511     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2512
2513     my $cpucount = $cpuinfo->{cpus} || 1;
2514
2515     foreach my $vmid (keys %$list) {
2516         next if $opt_vmid && ($vmid ne $opt_vmid);
2517
2518         my $conf = PVE::QemuConfig->load_config($vmid);
2519
2520         my $d = { vmid => $vmid };
2521         $d->{pid} = $list->{$vmid}->{pid};
2522
2523         # fixme: better status?
2524         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2525
2526         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2527         if (defined($size)) {
2528             $d->{disk} = 0; # no info available
2529             $d->{maxdisk} = $size;
2530         } else {
2531             $d->{disk} = 0;
2532             $d->{maxdisk} = 0;
2533         }
2534
2535         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2536             * ($conf->{cores} || $defaults->{cores});
2537         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2538         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2539
2540         $d->{name} = $conf->{name} || "VM $vmid";
2541         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2542             : $defaults->{memory}*(1024*1024);
2543
2544         if ($conf->{balloon}) {
2545             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2546             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2547                 : $defaults->{shares};
2548         }
2549
2550         $d->{uptime} = 0;
2551         $d->{cpu} = 0;
2552         $d->{mem} = 0;
2553
2554         $d->{netout} = 0;
2555         $d->{netin} = 0;
2556
2557         $d->{diskread} = 0;
2558         $d->{diskwrite} = 0;
2559
2560         $d->{template} = PVE::QemuConfig->is_template($conf);
2561
2562         $d->{serial} = 1 if conf_has_serial($conf);
2563         $d->{lock} = $conf->{lock} if $conf->{lock};
2564         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2565
2566         $res->{$vmid} = $d;
2567     }
2568
2569     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2570     foreach my $dev (keys %$netdev) {
2571         next if $dev !~ m/^tap([1-9]\d*)i/;
2572         my $vmid = $1;
2573         my $d = $res->{$vmid};
2574         next if !$d;
2575
2576         $d->{netout} += $netdev->{$dev}->{receive};
2577         $d->{netin} += $netdev->{$dev}->{transmit};
2578
2579         if ($full) {
2580             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2581             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2582         }
2583
2584     }
2585
2586     my $ctime = gettimeofday;
2587
2588     foreach my $vmid (keys %$list) {
2589
2590         my $d = $res->{$vmid};
2591         my $pid = $d->{pid};
2592         next if !$pid;
2593
2594         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2595         next if !$pstat; # not running
2596
2597         my $used = $pstat->{utime} + $pstat->{stime};
2598
2599         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2600
2601         if ($pstat->{vsize}) {
2602             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2603         }
2604
2605         my $old = $last_proc_pid_stat->{$pid};
2606         if (!$old) {
2607             $last_proc_pid_stat->{$pid} = {
2608                 time => $ctime,
2609                 used => $used,
2610                 cpu => 0,
2611             };
2612             next;
2613         }
2614
2615         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2616
2617         if ($dtime > 1000) {
2618             my $dutime = $used -  $old->{used};
2619
2620             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2621             $last_proc_pid_stat->{$pid} = {
2622                 time => $ctime,
2623                 used => $used,
2624                 cpu => $d->{cpu},
2625             };
2626         } else {
2627             $d->{cpu} = $old->{cpu};
2628         }
2629     }
2630
2631     return $res if !$full;
2632
2633     my $qmpclient = PVE::QMPClient->new();
2634
2635     my $ballooncb = sub {
2636         my ($vmid, $resp) = @_;
2637
2638         my $info = $resp->{'return'};
2639         return if !$info->{max_mem};
2640
2641         my $d = $res->{$vmid};
2642
2643         # use memory assigned to VM
2644         $d->{maxmem} = $info->{max_mem};
2645         $d->{balloon} = $info->{actual};
2646
2647         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2648             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2649             $d->{freemem} = $info->{free_mem};
2650         }
2651
2652         $d->{ballooninfo} = $info;
2653     };
2654
2655     my $blockstatscb = sub {
2656         my ($vmid, $resp) = @_;
2657         my $data = $resp->{'return'} || [];
2658         my $totalrdbytes = 0;
2659         my $totalwrbytes = 0;
2660
2661         for my $blockstat (@$data) {
2662             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2663             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2664
2665             $blockstat->{device} =~ s/drive-//;
2666             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2667         }
2668         $res->{$vmid}->{diskread} = $totalrdbytes;
2669         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2670     };
2671
2672     my $statuscb = sub {
2673         my ($vmid, $resp) = @_;
2674
2675         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2676         # this fails if ballon driver is not loaded, so this must be
2677         # the last commnand (following command are aborted if this fails).
2678         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2679
2680         my $status = 'unknown';
2681         if (!defined($status = $resp->{'return'}->{status})) {
2682             warn "unable to get VM status\n";
2683             return;
2684         }
2685
2686         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2687     };
2688
2689     foreach my $vmid (keys %$list) {
2690         next if $opt_vmid && ($vmid ne $opt_vmid);
2691         next if !$res->{$vmid}->{pid}; # not running
2692         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2693     }
2694
2695     $qmpclient->queue_execute(undef, 2);
2696
2697     foreach my $vmid (keys %$list) {
2698         next if $opt_vmid && ($vmid ne $opt_vmid);
2699         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2700     }
2701
2702     return $res;
2703 }
2704
2705 sub conf_has_serial {
2706     my ($conf) = @_;
2707
2708     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2709         if ($conf->{"serial$i"}) {
2710             return 1;
2711         }
2712     }
2713
2714     return 0;
2715 }
2716
2717 sub conf_has_audio {
2718     my ($conf, $id) = @_;
2719
2720     $id //= 0;
2721     my $audio = $conf->{"audio$id"};
2722     return undef if !defined($audio);
2723
2724     my $audioproperties = parse_property_string($audio_fmt, $audio);
2725     my $audiodriver = $audioproperties->{driver} // 'spice';
2726
2727     return {
2728         dev => $audioproperties->{device},
2729         dev_id => "audiodev$id",
2730         backend => $audiodriver,
2731         backend_id => "$audiodriver-backend${id}",
2732     };
2733 }
2734
2735 sub audio_devs {
2736     my ($audio, $audiopciaddr, $machine_version) = @_;
2737
2738     my $devs = [];
2739
2740     my $id = $audio->{dev_id};
2741     my $audiodev = "";
2742     if (min_version($machine_version, 4, 2)) {
2743         $audiodev = ",audiodev=$audio->{backend_id}";
2744     }
2745
2746     if ($audio->{dev} eq 'AC97') {
2747         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2748     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2749         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2750         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2751         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2752     } else {
2753         die "unkown audio device '$audio->{dev}', implement me!";
2754     }
2755
2756     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2757
2758     return $devs;
2759 }
2760
2761 sub vga_conf_has_spice {
2762     my ($vga) = @_;
2763
2764     my $vgaconf = parse_vga($vga);
2765     my $vgatype = $vgaconf->{type};
2766     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2767
2768     return $1 || 1;
2769 }
2770
2771 sub is_native($) {
2772     my ($arch) = @_;
2773     return get_host_arch() eq $arch;
2774 }
2775
2776 sub get_vm_arch {
2777     my ($conf) = @_;
2778     return $conf->{arch} // get_host_arch();
2779 }
2780
2781 my $default_machines = {
2782     x86_64 => 'pc',
2783     aarch64 => 'virt',
2784 };
2785
2786 sub get_vm_machine {
2787     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2788
2789     my $machine = $forcemachine || $conf->{machine};
2790
2791     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2792         $arch //= 'x86_64';
2793         $machine ||= $default_machines->{$arch};
2794         if ($add_pve_version) {
2795             $kvmversion //= kvm_user_version();
2796             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2797             $machine .= "+pve$pvever";
2798         }
2799     }
2800
2801     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2802         # for version-pinned machines that do not include a pve-version (e.g.
2803         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2804         $machine .= '+pve0';
2805     }
2806
2807     return $machine;
2808 }
2809
2810 sub get_ovmf_files($) {
2811     my ($arch) = @_;
2812
2813     my $ovmf = $OVMF->{$arch}
2814         or die "no OVMF images known for architecture '$arch'\n";
2815
2816     return @$ovmf;
2817 }
2818
2819 my $Arch2Qemu = {
2820     aarch64 => '/usr/bin/qemu-system-aarch64',
2821     x86_64 => '/usr/bin/qemu-system-x86_64',
2822 };
2823 sub get_command_for_arch($) {
2824     my ($arch) = @_;
2825     return '/usr/bin/kvm' if is_native($arch);
2826
2827     my $cmd = $Arch2Qemu->{$arch}
2828         or die "don't know how to emulate architecture '$arch'\n";
2829     return $cmd;
2830 }
2831
2832 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2833 # to use in a QEMU command line (-cpu element), first array_intersect the result
2834 # of query_supported_ with query_understood_. This is necessary because:
2835 #
2836 # a) query_understood_ returns flags the host cannot use and
2837 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2838 #    flags, but CPU settings - with most of them being flags. Those settings
2839 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2840 #
2841 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2842 # expensive. If you need the value returned from this, you can get it much
2843 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2844 # $accel being 'kvm' or 'tcg'.
2845 #
2846 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2847 # changes, automatically populating pmxcfs.
2848 #
2849 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2850 # since kvm and tcg machines support different flags
2851 #
2852 sub query_supported_cpu_flags {
2853     my ($arch) = @_;
2854
2855     $arch //= get_host_arch();
2856     my $default_machine = $default_machines->{$arch};
2857
2858     my $flags = {};
2859
2860     # FIXME: Once this is merged, the code below should work for ARM as well:
2861     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2862     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2863         $arch eq "aarch64";
2864
2865     my $kvm_supported = defined(kvm_version());
2866     my $qemu_cmd = get_command_for_arch($arch);
2867     my $fakevmid = -1;
2868     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2869
2870     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2871     my $query_supported_run_qemu = sub {
2872         my ($kvm) = @_;
2873
2874         my $flags = {};
2875         my $cmd = [
2876             $qemu_cmd,
2877             '-machine', $default_machine,
2878             '-display', 'none',
2879             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2880             '-mon', 'chardev=qmp,mode=control',
2881             '-pidfile', $pidfile,
2882             '-S', '-daemonize'
2883         ];
2884
2885         if (!$kvm) {
2886             push @$cmd, '-accel', 'tcg';
2887         }
2888
2889         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2890         die "QEMU flag querying VM exited with code " . $rc if $rc;
2891
2892         eval {
2893             my $cmd_result = mon_cmd(
2894                 $fakevmid,
2895                 'query-cpu-model-expansion',
2896                 type => 'full',
2897                 model => { name => 'host' }
2898             );
2899
2900             my $props = $cmd_result->{model}->{props};
2901             foreach my $prop (keys %$props) {
2902                 next if $props->{$prop} ne '1';
2903                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2904                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2905                 # We only keep those with underscores, to match /proc/cpuinfo
2906                 $prop =~ s/\.|-/_/g;
2907                 $flags->{$prop} = 1;
2908             }
2909         };
2910         my $err = $@;
2911
2912         # force stop with 10 sec timeout and 'nocheck'
2913         # always stop, even if QMP failed
2914         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2915
2916         die $err if $err;
2917
2918         return [ sort keys %$flags ];
2919     };
2920
2921     # We need to query QEMU twice, since KVM and TCG have different supported flags
2922     PVE::QemuConfig->lock_config($fakevmid, sub {
2923         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2924         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2925
2926         if ($kvm_supported) {
2927             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2928             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2929         }
2930     });
2931
2932     return $flags;
2933 }
2934
2935 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2936 my $understood_cpu_flag_dir = "/usr/share/kvm";
2937 sub query_understood_cpu_flags {
2938     my $arch = get_host_arch();
2939     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2940
2941     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2942         if ! -e $filepath;
2943
2944     my $raw = file_get_contents($filepath);
2945     $raw =~ s/^\s+|\s+$//g;
2946     my @flags = split(/\s+/, $raw);
2947
2948     return \@flags;
2949 }
2950
2951 sub config_to_command {
2952     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2953
2954     my $cmd = [];
2955     my $globalFlags = [];
2956     my $machineFlags = [];
2957     my $rtcFlags = [];
2958     my $devices = [];
2959     my $pciaddr = '';
2960     my $bridges = {};
2961     my $ostype = $conf->{ostype};
2962     my $winversion = windows_version($ostype);
2963     my $kvm = $conf->{kvm};
2964     my $nodename = nodename();
2965
2966     my $arch = get_vm_arch($conf);
2967     my $kvm_binary = get_command_for_arch($arch);
2968     my $kvmver = kvm_user_version($kvm_binary);
2969
2970     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2971         $kvmver //= "undefined";
2972         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2973     }
2974
2975     my $add_pve_version = min_version($kvmver, 4, 1);
2976
2977     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2978     my $machine_version = extract_version($machine_type, $kvmver);
2979     $kvm //= 1 if is_native($arch);
2980
2981     $machine_version =~ m/(\d+)\.(\d+)/;
2982     my ($machine_major, $machine_minor) = ($1, $2);
2983
2984     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
2985         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
2986     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
2987         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
2988             ." please upgrade node '$nodename'\n"
2989     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2990         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2991         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
2992             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
2993             ." node '$nodename'\n";
2994     }
2995
2996     # if a specific +pve version is required for a feature, use $version_guard
2997     # instead of min_version to allow machines to be run with the minimum
2998     # required version
2999     my $required_pve_version = 0;
3000     my $version_guard = sub {
3001         my ($major, $minor, $pve) = @_;
3002         return 0 if !min_version($machine_version, $major, $minor, $pve);
3003         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3004         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3005         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3006         return 1;
3007     };
3008
3009     if ($kvm && !defined kvm_version()) {
3010         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3011             ." or enable in BIOS.\n";
3012     }
3013
3014     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3015     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3016     my $use_old_bios_files = undef;
3017     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3018
3019     my $cpuunits = defined($conf->{cpuunits}) ?
3020             $conf->{cpuunits} : $defaults->{cpuunits};
3021
3022     push @$cmd, $kvm_binary;
3023
3024     push @$cmd, '-id', $vmid;
3025
3026     my $vmname = $conf->{name} || "vm$vmid";
3027
3028     push @$cmd, '-name', $vmname;
3029
3030     my $use_virtio = 0;
3031
3032     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3033     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3034     push @$cmd, '-mon', "chardev=qmp,mode=control";
3035
3036     if (min_version($machine_version, 2, 12)) {
3037         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3038         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3039     }
3040
3041     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3042
3043     push @$cmd, '-daemonize';
3044
3045     if ($conf->{smbios1}) {
3046         my $smbios_conf = parse_smbios1($conf->{smbios1});
3047         if ($smbios_conf->{base64}) {
3048             # Do not pass base64 flag to qemu
3049             delete $smbios_conf->{base64};
3050             my $smbios_string = "";
3051             foreach my $key (keys %$smbios_conf) {
3052                 my $value;
3053                 if ($key eq "uuid") {
3054                     $value = $smbios_conf->{uuid}
3055                 } else {
3056                     $value = decode_base64($smbios_conf->{$key});
3057                 }
3058                 # qemu accepts any binary data, only commas need escaping by double comma
3059                 $value =~ s/,/,,/g;
3060                 $smbios_string .= "," . $key . "=" . $value if $value;
3061             }
3062             push @$cmd, '-smbios', "type=1" . $smbios_string;
3063         } else {
3064             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3065         }
3066     }
3067
3068     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3069         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3070         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3071
3072         my ($path, $format);
3073         if (my $efidisk = $conf->{efidisk0}) {
3074             my $d = parse_drive('efidisk0', $efidisk);
3075             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3076             $format = $d->{format};
3077             if ($storeid) {
3078                 $path = PVE::Storage::path($storecfg, $d->{file});
3079                 if (!defined($format)) {
3080                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3081                     $format = qemu_img_format($scfg, $volname);
3082                 }
3083             } else {
3084                 $path = $d->{file};
3085                 die "efidisk format must be specified\n"
3086                     if !defined($format);
3087             }
3088         } else {
3089             warn "no efidisk configured! Using temporary efivars disk.\n";
3090             $path = "/tmp/$vmid-ovmf.fd";
3091             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3092             $format = 'raw';
3093         }
3094
3095         my $size_str = "";
3096
3097         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3098             $size_str = ",size=" . (-s $ovmf_vars);
3099         }
3100
3101         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3102         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3103     }
3104
3105     # load q35 config
3106     if ($q35) {
3107         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3108         if (min_version($machine_version, 4, 0)) {
3109             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3110         } else {
3111             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3112         }
3113     }
3114
3115     if ($conf->{vmgenid}) {
3116         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3117     }
3118
3119     # add usb controllers
3120     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3121         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3122     push @$devices, @usbcontrollers if @usbcontrollers;
3123     my $vga = parse_vga($conf->{vga});
3124
3125     my $qxlnum = vga_conf_has_spice($conf->{vga});
3126     $vga->{type} = 'qxl' if $qxlnum;
3127
3128     if (!$vga->{type}) {
3129         if ($arch eq 'aarch64') {
3130             $vga->{type} = 'virtio';
3131         } elsif (min_version($machine_version, 2, 9)) {
3132             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3133         } else {
3134             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3135         }
3136     }
3137
3138     # enable absolute mouse coordinates (needed by vnc)
3139     my $tablet;
3140     if (defined($conf->{tablet})) {
3141         $tablet = $conf->{tablet};
3142     } else {
3143         $tablet = $defaults->{tablet};
3144         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3145         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3146     }
3147
3148     if ($tablet) {
3149         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3150         my $kbd = print_keyboarddevice_full($conf, $arch);
3151         push @$devices, '-device', $kbd if defined($kbd);
3152     }
3153
3154     # host pci device passthrough
3155     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3156         $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type);
3157
3158     # usb devices
3159     my $usb_dev_features = {};
3160     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3161
3162     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3163         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3164     push @$devices, @usbdevices if @usbdevices;
3165     # serial devices
3166     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3167         if (my $path = $conf->{"serial$i"}) {
3168             if ($path eq 'socket') {
3169                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3170                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3171                 # On aarch64, serial0 is the UART device. Qemu only allows
3172                 # connecting UART devices via the '-serial' command line, as
3173                 # the device has a fixed slot on the hardware...
3174                 if ($arch eq 'aarch64' && $i == 0) {
3175                     push @$devices, '-serial', "chardev:serial$i";
3176                 } else {
3177                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3178                 }
3179             } else {
3180                 die "no such serial device\n" if ! -c $path;
3181                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3182                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3183             }
3184         }
3185     }
3186
3187     # parallel devices
3188     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3189         if (my $path = $conf->{"parallel$i"}) {
3190             die "no such parallel device\n" if ! -c $path;
3191             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3192             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3193             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3194         }
3195     }
3196
3197     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3198         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3199         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3200         push @$devices, @$audio_devs;
3201     }
3202
3203     my $sockets = 1;
3204     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3205     $sockets = $conf->{sockets} if  $conf->{sockets};
3206
3207     my $cores = $conf->{cores} || 1;
3208
3209     my $maxcpus = $sockets * $cores;
3210
3211     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3212
3213     my $allowed_vcpus = $cpuinfo->{cpus};
3214
3215     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3216         if ($allowed_vcpus < $maxcpus);
3217
3218     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3219
3220         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3221         for (my $i = 2; $i <= $vcpus; $i++)  {
3222             my $cpustr = print_cpu_device($conf,$i);
3223             push @$cmd, '-device', $cpustr;
3224         }
3225
3226     } else {
3227
3228         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3229     }
3230     push @$cmd, '-nodefaults';
3231
3232     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3233
3234     my $bootindex_hash = {};
3235     my $i = 1;
3236     foreach my $o (split(//, $bootorder)) {
3237         $bootindex_hash->{$o} = $i*100;
3238         $i++;
3239     }
3240
3241     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3242
3243     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3244
3245     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3246
3247     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3248         push @$devices, '-device', print_vga_device(
3249             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3250         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3251         push @$cmd,  '-vnc', "unix:$socket,password";
3252     } else {
3253         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3254         push @$cmd, '-nographic';
3255     }
3256
3257     # time drift fix
3258     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3259     my $useLocaltime = $conf->{localtime};
3260
3261     if ($winversion >= 5) { # windows
3262         $useLocaltime = 1 if !defined($conf->{localtime});
3263
3264         # use time drift fix when acpi is enabled
3265         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3266             $tdf = 1 if !defined($conf->{tdf});
3267         }
3268     }
3269
3270     if ($winversion >= 6) {
3271         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3272         push @$cmd, '-no-hpet';
3273     }
3274
3275     push @$rtcFlags, 'driftfix=slew' if $tdf;
3276
3277     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3278         push @$rtcFlags, "base=$conf->{startdate}";
3279     } elsif ($useLocaltime) {
3280         push @$rtcFlags, 'base=localtime';
3281     }
3282
3283     if ($forcecpu) {
3284         push @$cmd, '-cpu', $forcecpu;
3285     } else {
3286         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3287     }
3288
3289     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3290
3291     push @$cmd, '-S' if $conf->{freeze};
3292
3293     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3294
3295     my $guest_agent = parse_guest_agent($conf);
3296
3297     if ($guest_agent->{enabled}) {
3298         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3299         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3300
3301         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3302             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3303             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3304             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3305         } elsif ($guest_agent->{type} eq 'isa') {
3306             push @$devices, '-device', "isa-serial,chardev=qga0";
3307         }
3308     }
3309
3310     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3311     if ($rng && &$version_guard(4, 1, 2)) {
3312         check_rng_source($rng->{source});
3313
3314         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3315         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3316         my $limiter_str = "";
3317         if ($max_bytes) {
3318             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3319         }
3320
3321         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3322         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3323         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3324     }
3325
3326     my $spice_port;
3327
3328     if ($qxlnum) {
3329         if ($qxlnum > 1) {
3330             if ($winversion){
3331                 for (my $i = 1; $i < $qxlnum; $i++){
3332                     push @$devices, '-device', print_vga_device(
3333                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3334                 }
3335             } else {
3336                 # assume other OS works like Linux
3337                 my ($ram, $vram) = ("134217728", "67108864");
3338                 if ($vga->{memory}) {
3339                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3340                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3341                 }
3342                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3343                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3344             }
3345         }
3346
3347         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3348
3349         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3350         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3351         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3352
3353         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3354         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3355         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3356
3357         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3358         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3359
3360         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3361         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3362         if ($spice_enhancement->{foldersharing}) {
3363             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3364             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3365         }
3366
3367         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3368         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3369             if $spice_enhancement->{videostreaming};
3370
3371         push @$devices, '-spice', "$spice_opts";
3372     }
3373
3374     # enable balloon by default, unless explicitly disabled
3375     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3376         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3377         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3378     }
3379
3380     if ($conf->{watchdog}) {
3381         my $wdopts = parse_watchdog($conf->{watchdog});
3382         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3383         my $watchdog = $wdopts->{model} || 'i6300esb';
3384         push @$devices, '-device', "$watchdog$pciaddr";
3385         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3386     }
3387
3388     my $vollist = [];
3389     my $scsicontroller = {};
3390     my $ahcicontroller = {};
3391     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3392
3393     # Add iscsi initiator name if available
3394     if (my $initiator = get_initiator_name()) {
3395         push @$devices, '-iscsi', "initiator-name=$initiator";
3396     }
3397
3398     PVE::QemuConfig->foreach_volume($conf, sub {
3399         my ($ds, $drive) = @_;
3400
3401         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3402             push @$vollist, $drive->{file};
3403         }
3404
3405         # ignore efidisk here, already added in bios/fw handling code above
3406         return if $drive->{interface} eq 'efidisk';
3407
3408         $use_virtio = 1 if $ds =~ m/^virtio/;
3409
3410         if (drive_is_cdrom ($drive)) {
3411             if ($bootindex_hash->{d}) {
3412                 $drive->{bootindex} = $bootindex_hash->{d};
3413                 $bootindex_hash->{d} += 1;
3414             }
3415         } else {
3416             if ($bootindex_hash->{c}) {
3417                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3418                 $bootindex_hash->{c} += 1;
3419             }
3420         }
3421
3422         if ($drive->{interface} eq 'virtio'){
3423            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3424         }
3425
3426         if ($drive->{interface} eq 'scsi') {
3427
3428             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3429
3430             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3431                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3432
3433             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3434             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3435
3436             my $iothread = '';
3437             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3438                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3439                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3440             } elsif ($drive->{iothread}) {
3441                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3442             }
3443
3444             my $queues = '';
3445             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3446                 $queues = ",num_queues=$drive->{queues}";
3447             }
3448
3449             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3450                 if !$scsicontroller->{$controller};
3451             $scsicontroller->{$controller}=1;
3452         }
3453
3454         if ($drive->{interface} eq 'sata') {
3455             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3456             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3457             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3458                 if !$ahcicontroller->{$controller};
3459             $ahcicontroller->{$controller}=1;
3460         }
3461
3462         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3463         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3464
3465         push @$devices, '-drive',$drive_cmd;
3466         push @$devices, '-device', print_drivedevice_full(
3467             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3468     });
3469
3470     for (my $i = 0; $i < $MAX_NETS; $i++) {
3471          next if !$conf->{"net$i"};
3472          my $d = parse_net($conf->{"net$i"});
3473          next if !$d;
3474
3475          $use_virtio = 1 if $d->{model} eq 'virtio';
3476
3477          if ($bootindex_hash->{n}) {
3478             $d->{bootindex} = $bootindex_hash->{n};
3479             $bootindex_hash->{n} += 1;
3480          }
3481
3482          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3483          push @$devices, '-netdev', $netdevfull;
3484
3485          my $netdevicefull = print_netdevice_full(
3486             $vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3487
3488          push @$devices, '-device', $netdevicefull;
3489     }
3490
3491     if ($conf->{ivshmem}) {
3492         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3493
3494         my $bus;
3495         if ($q35) {
3496             $bus = print_pcie_addr("ivshmem");
3497         } else {
3498             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3499         }
3500
3501         my $ivshmem_name = $ivshmem->{name} // $vmid;
3502         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3503
3504         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3505         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3506             .",size=$ivshmem->{size}M";
3507     }
3508
3509     # pci.4 is nested in pci.1
3510     $bridges->{1} = 1 if $bridges->{4};
3511
3512     if (!$q35) {
3513         # add pci bridges
3514         if (min_version($machine_version, 2, 3)) {
3515            $bridges->{1} = 1;
3516            $bridges->{2} = 1;
3517         }
3518
3519         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3520
3521     }
3522
3523     for my $k (sort {$b cmp $a} keys %$bridges) {
3524         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3525
3526         my $k_name = $k;
3527         if ($k == 2 && $legacy_igd) {
3528             $k_name = "$k-igd";
3529         }
3530         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3531
3532         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3533         if ($q35) {
3534             # add after -readconfig pve-q35.cfg
3535             splice @$devices, 2, 0, '-device', $devstr;
3536         } else {
3537             unshift @$devices, '-device', $devstr if $k > 0;
3538         }
3539     }
3540
3541     if (!$kvm) {
3542         push @$machineFlags, 'accel=tcg';
3543     }
3544
3545     my $machine_type_min = $machine_type;
3546     if ($add_pve_version) {
3547         $machine_type_min =~ s/\+pve\d+$//;
3548         $machine_type_min .= "+pve$required_pve_version";
3549     }
3550     push @$machineFlags, "type=${machine_type_min}";
3551
3552     push @$cmd, @$devices;
3553     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3554     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3555     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3556
3557     if (my $vmstate = $conf->{vmstate}) {
3558         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3559         push @$vollist, $vmstate;
3560         push @$cmd, '-loadstate', $statepath;
3561         print "activating and using '$vmstate' as vmstate\n";
3562     }
3563
3564     # add custom args
3565     if ($conf->{args}) {
3566         my $aa = PVE::Tools::split_args($conf->{args});
3567         push @$cmd, @$aa;
3568     }
3569
3570     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3571 }
3572
3573 sub check_rng_source {
3574     my ($source) = @_;
3575
3576     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3577     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3578         if ! -e $source;
3579
3580     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3581     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3582         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3583         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3584         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3585             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3586             ." to the host.\n";
3587     }
3588 }
3589
3590 sub spice_port {
3591     my ($vmid) = @_;
3592
3593     my $res = mon_cmd($vmid, 'query-spice');
3594
3595     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3596 }
3597
3598 sub vm_devices_list {
3599     my ($vmid) = @_;
3600
3601     my $res = mon_cmd($vmid, 'query-pci');
3602     my $devices_to_check = [];
3603     my $devices = {};
3604     foreach my $pcibus (@$res) {
3605         push @$devices_to_check, @{$pcibus->{devices}},
3606     }
3607
3608     while (@$devices_to_check) {
3609         my $to_check = [];
3610         for my $d (@$devices_to_check) {
3611             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3612             next if !$d->{'pci_bridge'};
3613
3614             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3615             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3616         }
3617         $devices_to_check = $to_check;
3618     }
3619
3620     my $resblock = mon_cmd($vmid, 'query-block');
3621     foreach my $block (@$resblock) {
3622         if($block->{device} =~ m/^drive-(\S+)/){
3623                 $devices->{$1} = 1;
3624         }
3625     }
3626
3627     my $resmice = mon_cmd($vmid, 'query-mice');
3628     foreach my $mice (@$resmice) {
3629         if ($mice->{name} eq 'QEMU HID Tablet') {
3630             $devices->{tablet} = 1;
3631             last;
3632         }
3633     }
3634
3635     # for usb devices there is no query-usb
3636     # but we can iterate over the entries in
3637     # qom-list path=/machine/peripheral
3638     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3639     foreach my $per (@$resperipheral) {
3640         if ($per->{name} =~ m/^usb\d+$/) {
3641             $devices->{$per->{name}} = 1;
3642         }
3643     }
3644
3645     return $devices;
3646 }
3647
3648 sub vm_deviceplug {
3649     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3650
3651     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3652
3653     my $devices_list = vm_devices_list($vmid);
3654     return 1 if defined($devices_list->{$deviceid});
3655
3656     # add PCI bridge if we need it for the device
3657     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3658
3659     if ($deviceid eq 'tablet') {
3660
3661         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3662
3663     } elsif ($deviceid eq 'keyboard') {
3664
3665         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3666
3667     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3668
3669         die "usb hotplug currently not reliable\n";
3670         # since we can't reliably hot unplug all added usb devices
3671         # and usb passthrough disables live migration
3672         # we disable usb hotplugging for now
3673         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3674
3675     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3676
3677         qemu_iothread_add($vmid, $deviceid, $device);
3678
3679         qemu_driveadd($storecfg, $vmid, $device);
3680         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3681
3682         qemu_deviceadd($vmid, $devicefull);
3683         eval { qemu_deviceaddverify($vmid, $deviceid); };
3684         if (my $err = $@) {
3685             eval { qemu_drivedel($vmid, $deviceid); };
3686             warn $@ if $@;
3687             die $err;
3688         }
3689
3690     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3691
3692
3693         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3694         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3695         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3696
3697         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3698
3699         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3700             qemu_iothread_add($vmid, $deviceid, $device);
3701             $devicefull .= ",iothread=iothread-$deviceid";
3702         }
3703
3704         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3705             $devicefull .= ",num_queues=$device->{queues}";
3706         }
3707
3708         qemu_deviceadd($vmid, $devicefull);
3709         qemu_deviceaddverify($vmid, $deviceid);
3710
3711     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3712
3713         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3714         qemu_driveadd($storecfg, $vmid, $device);
3715
3716         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3717         eval { qemu_deviceadd($vmid, $devicefull); };
3718         if (my $err = $@) {
3719             eval { qemu_drivedel($vmid, $deviceid); };
3720             warn $@ if $@;
3721             die $err;
3722         }
3723
3724     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3725
3726         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3727
3728         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3729         my $use_old_bios_files = undef;
3730         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3731
3732         my $netdevicefull = print_netdevice_full(
3733             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3734         qemu_deviceadd($vmid, $netdevicefull);
3735         eval {
3736             qemu_deviceaddverify($vmid, $deviceid);
3737             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3738         };
3739         if (my $err = $@) {
3740             eval { qemu_netdevdel($vmid, $deviceid); };
3741             warn $@ if $@;
3742             die $err;
3743         }
3744
3745     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3746
3747         my $bridgeid = $2;
3748         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3749         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3750
3751         qemu_deviceadd($vmid, $devicefull);
3752         qemu_deviceaddverify($vmid, $deviceid);
3753
3754     } else {
3755         die "can't hotplug device '$deviceid'\n";
3756     }
3757
3758     return 1;
3759 }
3760
3761 # fixme: this should raise exceptions on error!
3762 sub vm_deviceunplug {
3763     my ($vmid, $conf, $deviceid) = @_;
3764
3765     my $devices_list = vm_devices_list($vmid);
3766     return 1 if !defined($devices_list->{$deviceid});
3767
3768     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3769
3770     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3771
3772         qemu_devicedel($vmid, $deviceid);
3773
3774     } elsif ($deviceid =~ m/^usb\d+$/) {
3775
3776         die "usb hotplug currently not reliable\n";
3777         # when unplugging usb devices this way,
3778         # there may be remaining usb controllers/hubs
3779         # so we disable it for now
3780         qemu_devicedel($vmid, $deviceid);
3781         qemu_devicedelverify($vmid, $deviceid);
3782
3783     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3784
3785         qemu_devicedel($vmid, $deviceid);
3786         qemu_devicedelverify($vmid, $deviceid);
3787         qemu_drivedel($vmid, $deviceid);
3788         qemu_iothread_del($conf, $vmid, $deviceid);
3789
3790     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3791
3792         qemu_devicedel($vmid, $deviceid);
3793         qemu_devicedelverify($vmid, $deviceid);
3794         qemu_iothread_del($conf, $vmid, $deviceid);
3795
3796     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3797
3798         qemu_devicedel($vmid, $deviceid);
3799         qemu_drivedel($vmid, $deviceid);
3800         qemu_deletescsihw($conf, $vmid, $deviceid);
3801
3802     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3803
3804         qemu_devicedel($vmid, $deviceid);
3805         qemu_devicedelverify($vmid, $deviceid);
3806         qemu_netdevdel($vmid, $deviceid);
3807
3808     } else {
3809         die "can't unplug device '$deviceid'\n";
3810     }
3811
3812     return 1;
3813 }
3814
3815 sub qemu_deviceadd {
3816     my ($vmid, $devicefull) = @_;
3817
3818     $devicefull = "driver=".$devicefull;
3819     my %options =  split(/[=,]/, $devicefull);
3820
3821     mon_cmd($vmid, "device_add" , %options);
3822 }
3823
3824 sub qemu_devicedel {
3825     my ($vmid, $deviceid) = @_;
3826
3827     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3828 }
3829
3830 sub qemu_iothread_add {
3831     my($vmid, $deviceid, $device) = @_;
3832
3833     if ($device->{iothread}) {
3834         my $iothreads = vm_iothreads_list($vmid);
3835         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3836     }
3837 }
3838
3839 sub qemu_iothread_del {
3840     my($conf, $vmid, $deviceid) = @_;
3841
3842     my $confid = $deviceid;
3843     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3844         $confid = 'scsi' . $1;
3845     }
3846     my $device = parse_drive($confid, $conf->{$confid});
3847     if ($device->{iothread}) {
3848         my $iothreads = vm_iothreads_list($vmid);
3849         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3850     }
3851 }
3852
3853 sub qemu_objectadd {
3854     my($vmid, $objectid, $qomtype) = @_;
3855
3856     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3857
3858     return 1;
3859 }
3860
3861 sub qemu_objectdel {
3862     my($vmid, $objectid) = @_;
3863
3864     mon_cmd($vmid, "object-del", id => $objectid);
3865
3866     return 1;
3867 }
3868
3869 sub qemu_driveadd {
3870     my ($storecfg, $vmid, $device) = @_;
3871
3872     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3873     $drive =~ s/\\/\\\\/g;
3874     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3875
3876     # If the command succeeds qemu prints: "OK"
3877     return 1 if $ret =~ m/OK/s;
3878
3879     die "adding drive failed: $ret\n";
3880 }
3881
3882 sub qemu_drivedel {
3883     my($vmid, $deviceid) = @_;
3884
3885     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3886     $ret =~ s/^\s+//;
3887
3888     return 1 if $ret eq "";
3889
3890     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3891     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3892
3893     die "deleting drive $deviceid failed : $ret\n";
3894 }
3895
3896 sub qemu_deviceaddverify {
3897     my ($vmid, $deviceid) = @_;
3898
3899     for (my $i = 0; $i <= 5; $i++) {
3900          my $devices_list = vm_devices_list($vmid);
3901          return 1 if defined($devices_list->{$deviceid});
3902          sleep 1;
3903     }
3904
3905     die "error on hotplug device '$deviceid'\n";
3906 }
3907
3908
3909 sub qemu_devicedelverify {
3910     my ($vmid, $deviceid) = @_;
3911
3912     # need to verify that the device is correctly removed as device_del
3913     # is async and empty return is not reliable
3914
3915     for (my $i = 0; $i <= 5; $i++) {
3916          my $devices_list = vm_devices_list($vmid);
3917          return 1 if !defined($devices_list->{$deviceid});
3918          sleep 1;
3919     }
3920
3921     die "error on hot-unplugging device '$deviceid'\n";
3922 }
3923
3924 sub qemu_findorcreatescsihw {
3925     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3926
3927     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3928
3929     my $scsihwid="$controller_prefix$controller";
3930     my $devices_list = vm_devices_list($vmid);
3931
3932     if(!defined($devices_list->{$scsihwid})) {
3933         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3934     }
3935
3936     return 1;
3937 }
3938
3939 sub qemu_deletescsihw {
3940     my ($conf, $vmid, $opt) = @_;
3941
3942     my $device = parse_drive($opt, $conf->{$opt});
3943
3944     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3945         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3946         return 1;
3947     }
3948
3949     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3950
3951     my $devices_list = vm_devices_list($vmid);
3952     foreach my $opt (keys %{$devices_list}) {
3953         if (is_valid_drivename($opt)) {
3954             my $drive = parse_drive($opt, $conf->{$opt});
3955             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3956                 return 1;
3957             }
3958         }
3959     }
3960
3961     my $scsihwid="scsihw$controller";
3962
3963     vm_deviceunplug($vmid, $conf, $scsihwid);
3964
3965     return 1;
3966 }
3967
3968 sub qemu_add_pci_bridge {
3969     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3970
3971     my $bridges = {};
3972
3973     my $bridgeid;
3974
3975     print_pci_addr($device, $bridges, $arch, $machine_type);
3976
3977     while (my ($k, $v) = each %$bridges) {
3978         $bridgeid = $k;
3979     }
3980     return 1 if !defined($bridgeid) || $bridgeid < 1;
3981
3982     my $bridge = "pci.$bridgeid";
3983     my $devices_list = vm_devices_list($vmid);
3984
3985     if (!defined($devices_list->{$bridge})) {
3986         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3987     }
3988
3989     return 1;
3990 }
3991
3992 sub qemu_set_link_status {
3993     my ($vmid, $device, $up) = @_;
3994
3995     mon_cmd($vmid, "set_link", name => $device,
3996                up => $up ? JSON::true : JSON::false);
3997 }
3998
3999 sub qemu_netdevadd {
4000     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4001
4002     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4003     my %options =  split(/[=,]/, $netdev);
4004
4005     if (defined(my $vhost = $options{vhost})) {
4006         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4007     }
4008
4009     if (defined(my $queues = $options{queues})) {
4010         $options{queues} = $queues + 0;
4011     }
4012
4013     mon_cmd($vmid, "netdev_add",  %options);
4014     return 1;
4015 }
4016
4017 sub qemu_netdevdel {
4018     my ($vmid, $deviceid) = @_;
4019
4020     mon_cmd($vmid, "netdev_del", id => $deviceid);
4021 }
4022
4023 sub qemu_usb_hotplug {
4024     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4025
4026     return if !$device;
4027
4028     # remove the old one first
4029     vm_deviceunplug($vmid, $conf, $deviceid);
4030
4031     # check if xhci controller is necessary and available
4032     if ($device->{usb3}) {
4033
4034         my $devicelist = vm_devices_list($vmid);
4035
4036         if (!$devicelist->{xhci}) {
4037             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4038             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4039         }
4040     }
4041     my $d = parse_usb_device($device->{host});
4042     $d->{usb3} = $device->{usb3};
4043
4044     # add the new one
4045     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4046 }
4047
4048 sub qemu_cpu_hotplug {
4049     my ($vmid, $conf, $vcpus) = @_;
4050
4051     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4052
4053     my $sockets = 1;
4054     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4055     $sockets = $conf->{sockets} if  $conf->{sockets};
4056     my $cores = $conf->{cores} || 1;
4057     my $maxcpus = $sockets * $cores;
4058
4059     $vcpus = $maxcpus if !$vcpus;
4060
4061     die "you can't add more vcpus than maxcpus\n"
4062         if $vcpus > $maxcpus;
4063
4064     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4065
4066     if ($vcpus < $currentvcpus) {
4067
4068         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4069
4070             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4071                 qemu_devicedel($vmid, "cpu$i");
4072                 my $retry = 0;
4073                 my $currentrunningvcpus = undef;
4074                 while (1) {
4075                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4076                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4077                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4078                     $retry++;
4079                     sleep 1;
4080                 }
4081                 #update conf after each succesfull cpu unplug
4082                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4083                 PVE::QemuConfig->write_config($vmid, $conf);
4084             }
4085         } else {
4086             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4087         }
4088
4089         return;
4090     }
4091
4092     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4093     die "vcpus in running vm does not match its configuration\n"
4094         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4095
4096     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4097
4098         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4099             my $cpustr = print_cpu_device($conf, $i);
4100             qemu_deviceadd($vmid, $cpustr);
4101
4102             my $retry = 0;
4103             my $currentrunningvcpus = undef;
4104             while (1) {
4105                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4106                 last if scalar(@{$currentrunningvcpus}) == $i;
4107                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4108                 sleep 1;
4109                 $retry++;
4110             }
4111             #update conf after each succesfull cpu hotplug
4112             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4113             PVE::QemuConfig->write_config($vmid, $conf);
4114         }
4115     } else {
4116
4117         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4118             mon_cmd($vmid, "cpu-add", id => int($i));
4119         }
4120     }
4121 }
4122
4123 sub qemu_block_set_io_throttle {
4124     my ($vmid, $deviceid,
4125         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4126         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4127         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4128         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4129
4130     return if !check_running($vmid) ;
4131
4132     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4133         bps => int($bps),
4134         bps_rd => int($bps_rd),
4135         bps_wr => int($bps_wr),
4136         iops => int($iops),
4137         iops_rd => int($iops_rd),
4138         iops_wr => int($iops_wr),
4139         bps_max => int($bps_max),
4140         bps_rd_max => int($bps_rd_max),
4141         bps_wr_max => int($bps_wr_max),
4142         iops_max => int($iops_max),
4143         iops_rd_max => int($iops_rd_max),
4144         iops_wr_max => int($iops_wr_max),
4145         bps_max_length => int($bps_max_length),
4146         bps_rd_max_length => int($bps_rd_max_length),
4147         bps_wr_max_length => int($bps_wr_max_length),
4148         iops_max_length => int($iops_max_length),
4149         iops_rd_max_length => int($iops_rd_max_length),
4150         iops_wr_max_length => int($iops_wr_max_length),
4151     );
4152
4153 }
4154
4155 # old code, only used to shutdown old VM after update
4156 sub __read_avail {
4157     my ($fh, $timeout) = @_;
4158
4159     my $sel = new IO::Select;
4160     $sel->add($fh);
4161
4162     my $res = '';
4163     my $buf;
4164
4165     my @ready;
4166     while (scalar (@ready = $sel->can_read($timeout))) {
4167         my $count;
4168         if ($count = $fh->sysread($buf, 8192)) {
4169             if ($buf =~ /^(.*)\(qemu\) $/s) {
4170                 $res .= $1;
4171                 last;
4172             } else {
4173                 $res .= $buf;
4174             }
4175         } else {
4176             if (!defined($count)) {
4177                 die "$!\n";
4178             }
4179             last;
4180         }
4181     }
4182
4183     die "monitor read timeout\n" if !scalar(@ready);
4184
4185     return $res;
4186 }
4187
4188 sub qemu_block_resize {
4189     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4190
4191     my $running = check_running($vmid);
4192
4193     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4194
4195     return if !$running;
4196
4197     my $padding = (1024 - $size % 1024) % 1024;
4198     $size = $size + $padding;
4199
4200     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4201
4202 }
4203
4204 sub qemu_volume_snapshot {
4205     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4206
4207     my $running = check_running($vmid);
4208
4209     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4210         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4211     } else {
4212         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4213     }
4214 }
4215
4216 sub qemu_volume_snapshot_delete {
4217     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4218
4219     my $running = check_running($vmid);
4220
4221     if($running) {
4222
4223         $running = undef;
4224         my $conf = PVE::QemuConfig->load_config($vmid);
4225         PVE::QemuConfig->foreach_volume($conf, sub {
4226             my ($ds, $drive) = @_;
4227             $running = 1 if $drive->{file} eq $volid;
4228         });
4229     }
4230
4231     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4232         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4233     } else {
4234         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4235     }
4236 }
4237
4238 sub set_migration_caps {
4239     my ($vmid) = @_;
4240
4241     my $cap_ref = [];
4242
4243     my $enabled_cap = {
4244         "auto-converge" => 1,
4245         "xbzrle" => 1,
4246         "x-rdma-pin-all" => 0,
4247         "zero-blocks" => 0,
4248         "compress" => 0
4249     };
4250
4251     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4252
4253     for my $supported_capability (@$supported_capabilities) {
4254         push @$cap_ref, {
4255             capability => $supported_capability->{capability},
4256             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4257         };
4258     }
4259
4260     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4261 }
4262
4263 sub foreach_volid {
4264     my ($conf, $func, @param) = @_;
4265
4266     my $volhash = {};
4267
4268     my $test_volid = sub {
4269         my ($key, $drive, $snapname) = @_;
4270
4271         my $volid = $drive->{file};
4272         return if !$volid;
4273
4274         $volhash->{$volid}->{cdrom} //= 1;
4275         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4276
4277         my $replicate = $drive->{replicate} // 1;
4278         $volhash->{$volid}->{replicate} //= 0;
4279         $volhash->{$volid}->{replicate} = 1 if $replicate;
4280
4281         $volhash->{$volid}->{shared} //= 0;
4282         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4283
4284         $volhash->{$volid}->{referenced_in_config} //= 0;
4285         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4286
4287         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4288             if defined($snapname);
4289
4290         my $size = $drive->{size};
4291         $volhash->{$volid}->{size} //= $size if $size;
4292
4293         $volhash->{$volid}->{is_vmstate} //= 0;
4294         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4295
4296         $volhash->{$volid}->{is_unused} //= 0;
4297         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4298     };
4299
4300     my $include_opts = {
4301         extra_keys => ['vmstate'],
4302         include_unused => 1,
4303     };
4304
4305     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4306     foreach my $snapname (keys %{$conf->{snapshots}}) {
4307         my $snap = $conf->{snapshots}->{$snapname};
4308         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4309     }
4310
4311     foreach my $volid (keys %$volhash) {
4312         &$func($volid, $volhash->{$volid}, @param);
4313     }
4314 }
4315
4316 my $fast_plug_option = {
4317     'lock' => 1,
4318     'name' => 1,
4319     'onboot' => 1,
4320     'shares' => 1,
4321     'startup' => 1,
4322     'description' => 1,
4323     'protection' => 1,
4324     'vmstatestorage' => 1,
4325     'hookscript' => 1,
4326     'tags' => 1,
4327 };
4328
4329 # hotplug changes in [PENDING]
4330 # $selection hash can be used to only apply specified options, for
4331 # example: { cores => 1 } (only apply changed 'cores')
4332 # $errors ref is used to return error messages
4333 sub vmconfig_hotplug_pending {
4334     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4335
4336     my $defaults = load_defaults();
4337     my $arch = get_vm_arch($conf);
4338     my $machine_type = get_vm_machine($conf, undef, $arch);
4339
4340     # commit values which do not have any impact on running VM first
4341     # Note: those option cannot raise errors, we we do not care about
4342     # $selection and always apply them.
4343
4344     my $add_error = sub {
4345         my ($opt, $msg) = @_;
4346         $errors->{$opt} = "hotplug problem - $msg";
4347     };
4348
4349     my $changes = 0;
4350     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4351         if ($fast_plug_option->{$opt}) {
4352             $conf->{$opt} = $conf->{pending}->{$opt};
4353             delete $conf->{pending}->{$opt};
4354             $changes = 1;
4355         }
4356     }
4357
4358     if ($changes) {
4359         PVE::QemuConfig->write_config($vmid, $conf);
4360     }
4361
4362     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4363
4364     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4365     foreach my $opt (sort keys %$pending_delete_hash) {
4366         next if $selection && !$selection->{$opt};
4367         my $force = $pending_delete_hash->{$opt}->{force};
4368         eval {
4369             if ($opt eq 'hotplug') {
4370                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4371             } elsif ($opt eq 'tablet') {
4372                 die "skip\n" if !$hotplug_features->{usb};
4373                 if ($defaults->{tablet}) {
4374                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4375                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4376                         if $arch eq 'aarch64';
4377                 } else {
4378                     vm_deviceunplug($vmid, $conf, 'tablet');
4379                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4380                 }
4381             } elsif ($opt =~ m/^usb\d+/) {
4382                 die "skip\n";
4383                 # since we cannot reliably hot unplug usb devices
4384                 # we are disabling it
4385                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4386                 vm_deviceunplug($vmid, $conf, $opt);
4387             } elsif ($opt eq 'vcpus') {
4388                 die "skip\n" if !$hotplug_features->{cpu};
4389                 qemu_cpu_hotplug($vmid, $conf, undef);
4390             } elsif ($opt eq 'balloon') {
4391                 # enable balloon device is not hotpluggable
4392                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4393                 # here we reset the ballooning value to memory
4394                 my $balloon = $conf->{memory} || $defaults->{memory};
4395                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4396             } elsif ($fast_plug_option->{$opt}) {
4397                 # do nothing
4398             } elsif ($opt =~ m/^net(\d+)$/) {
4399                 die "skip\n" if !$hotplug_features->{network};
4400                 vm_deviceunplug($vmid, $conf, $opt);
4401             } elsif (is_valid_drivename($opt)) {
4402                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4403                 vm_deviceunplug($vmid, $conf, $opt);
4404                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4405             } elsif ($opt =~ m/^memory$/) {
4406                 die "skip\n" if !$hotplug_features->{memory};
4407                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4408             } elsif ($opt eq 'cpuunits') {
4409                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4410             } elsif ($opt eq 'cpulimit') {
4411                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4412             } else {
4413                 die "skip\n";
4414             }
4415         };
4416         if (my $err = $@) {
4417             &$add_error($opt, $err) if $err ne "skip\n";
4418         } else {
4419             delete $conf->{$opt};
4420             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4421         }
4422     }
4423
4424     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4425     $apply_pending_cloudinit = sub {
4426         return if $apply_pending_cloudinit_done; # once is enough
4427         $apply_pending_cloudinit_done = 1; # once is enough
4428
4429         my ($key, $value) = @_;
4430
4431         my @cloudinit_opts = keys %$confdesc_cloudinit;
4432         foreach my $opt (keys %{$conf->{pending}}) {
4433             next if !grep { $_ eq $opt } @cloudinit_opts;
4434             $conf->{$opt} = delete $conf->{pending}->{$opt};
4435         }
4436
4437         my $new_conf = { %$conf };
4438         $new_conf->{$key} = $value;
4439         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4440     };
4441
4442     foreach my $opt (keys %{$conf->{pending}}) {
4443         next if $selection && !$selection->{$opt};
4444         my $value = $conf->{pending}->{$opt};
4445         eval {
4446             if ($opt eq 'hotplug') {
4447                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4448             } elsif ($opt eq 'tablet') {
4449                 die "skip\n" if !$hotplug_features->{usb};
4450                 if ($value == 1) {
4451                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4452                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4453                         if $arch eq 'aarch64';
4454                 } elsif ($value == 0) {
4455                     vm_deviceunplug($vmid, $conf, 'tablet');
4456                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4457                 }
4458             } elsif ($opt =~ m/^usb\d+$/) {
4459                 die "skip\n";
4460                 # since we cannot reliably hot unplug usb devices
4461                 # we are disabling it
4462                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4463                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4464                 die "skip\n" if !$d;
4465                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4466             } elsif ($opt eq 'vcpus') {
4467                 die "skip\n" if !$hotplug_features->{cpu};
4468                 qemu_cpu_hotplug($vmid, $conf, $value);
4469             } elsif ($opt eq 'balloon') {
4470                 # enable/disable balloning device is not hotpluggable
4471                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4472                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4473                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4474
4475                 # allow manual ballooning if shares is set to zero
4476                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4477                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4478                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4479                 }
4480             } elsif ($opt =~ m/^net(\d+)$/) {
4481                 # some changes can be done without hotplug
4482                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4483                                     $vmid, $opt, $value, $arch, $machine_type);
4484             } elsif (is_valid_drivename($opt)) {
4485                 die "skip\n" if $opt eq 'efidisk0';
4486                 # some changes can be done without hotplug
4487                 my $drive = parse_drive($opt, $value);
4488                 if (drive_is_cloudinit($drive)) {
4489                     &$apply_pending_cloudinit($opt, $value);
4490                 }
4491                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4492                                      $vmid, $opt, $value, $arch, $machine_type);
4493             } elsif ($opt =~ m/^memory$/) { #dimms
4494                 die "skip\n" if !$hotplug_features->{memory};
4495                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4496             } elsif ($opt eq 'cpuunits') {
4497                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4498             } elsif ($opt eq 'cpulimit') {
4499                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4500                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4501             } else {
4502                 die "skip\n";  # skip non-hot-pluggable options
4503             }
4504         };
4505         if (my $err = $@) {
4506             &$add_error($opt, $err) if $err ne "skip\n";
4507         } else {
4508             $conf->{$opt} = $value;
4509             delete $conf->{pending}->{$opt};
4510         }
4511     }
4512
4513     PVE::QemuConfig->write_config($vmid, $conf);
4514 }
4515
4516 sub try_deallocate_drive {
4517     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4518
4519     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4520         my $volid = $drive->{file};
4521         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4522             my $sid = PVE::Storage::parse_volume_id($volid);
4523             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4524
4525             # check if the disk is really unused
4526             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4527                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4528             PVE::Storage::vdisk_free($storecfg, $volid);
4529             return 1;
4530         } else {
4531             # If vm is not owner of this disk remove from config
4532             return 1;
4533         }
4534     }
4535
4536     return undef;
4537 }
4538
4539 sub vmconfig_delete_or_detach_drive {
4540     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4541
4542     my $drive = parse_drive($opt, $conf->{$opt});
4543
4544     my $rpcenv = PVE::RPCEnvironment::get();
4545     my $authuser = $rpcenv->get_user();
4546
4547     if ($force) {
4548         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4549         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4550     } else {
4551         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4552     }
4553 }
4554
4555
4556
4557 sub vmconfig_apply_pending {
4558     my ($vmid, $conf, $storecfg, $errors) = @_;
4559
4560     my $add_apply_error = sub {
4561         my ($opt, $msg) = @_;
4562         my $err_msg = "unable to apply pending change $opt : $msg";
4563         $errors->{$opt} = $err_msg;
4564         warn $err_msg;
4565     };
4566
4567     # cold plug
4568
4569     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4570     foreach my $opt (sort keys %$pending_delete_hash) {
4571         my $force = $pending_delete_hash->{$opt}->{force};
4572         eval {
4573             if ($opt =~ m/^unused/) {
4574                 die "internal error";
4575             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4576                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4577             }
4578         };
4579         if (my $err = $@) {
4580             $add_apply_error->($opt, $err);
4581         } else {
4582             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4583             delete $conf->{$opt};
4584         }
4585     }
4586
4587     PVE::QemuConfig->cleanup_pending($conf);
4588
4589     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4590         next if $opt eq 'delete'; # just to be sure
4591         eval {
4592             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4593                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4594             }
4595         };
4596         if (my $err = $@) {
4597             $add_apply_error->($opt, $err);
4598         } else {
4599             $conf->{$opt} = delete $conf->{pending}->{$opt};
4600         }
4601     }
4602
4603     # write all changes at once to avoid unnecessary i/o
4604     PVE::QemuConfig->write_config($vmid, $conf);
4605 }
4606
4607 sub vmconfig_update_net {
4608     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4609
4610     my $newnet = parse_net($value);
4611
4612     if ($conf->{$opt}) {
4613         my $oldnet = parse_net($conf->{$opt});
4614
4615         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4616             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4617             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4618             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4619
4620             # for non online change, we try to hot-unplug
4621             die "skip\n" if !$hotplug;
4622             vm_deviceunplug($vmid, $conf, $opt);
4623         } else {
4624
4625             die "internal error" if $opt !~ m/net(\d+)/;
4626             my $iface = "tap${vmid}i$1";
4627
4628             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4629                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4630                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4631                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4632                 PVE::Network::tap_unplug($iface);
4633
4634                 if ($have_sdn) {
4635                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4636                 } else {
4637                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4638                 }
4639             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4640                 # Rate can be applied on its own but any change above needs to
4641                 # include the rate in tap_plug since OVS resets everything.
4642                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4643             }
4644
4645             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4646                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4647             }
4648
4649             return 1;
4650         }
4651     }
4652
4653     if ($hotplug) {
4654         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4655     } else {
4656         die "skip\n";
4657     }
4658 }
4659
4660 sub vmconfig_update_disk {
4661     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4662
4663     my $drive = parse_drive($opt, $value);
4664
4665     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4666         my $media = $drive->{media} || 'disk';
4667         my $oldmedia = $old_drive->{media} || 'disk';
4668         die "unable to change media type\n" if $media ne $oldmedia;
4669
4670         if (!drive_is_cdrom($old_drive)) {
4671
4672             if ($drive->{file} ne $old_drive->{file}) {
4673
4674                 die "skip\n" if !$hotplug;
4675
4676                 # unplug and register as unused
4677                 vm_deviceunplug($vmid, $conf, $opt);
4678                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4679
4680             } else {
4681                 # update existing disk
4682
4683                 # skip non hotpluggable value
4684                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4685                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4686                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4687                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4688                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4689                     die "skip\n";
4690                 }
4691
4692                 # apply throttle
4693                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4694                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4695                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4696                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4697                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4698                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4699                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4700                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4701                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4702                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4703                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4704                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4705                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4706                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4707                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4708                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4709                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4710                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4711
4712                     qemu_block_set_io_throttle(
4713                         $vmid,"drive-$opt",
4714                         ($drive->{mbps} || 0)*1024*1024,
4715                         ($drive->{mbps_rd} || 0)*1024*1024,
4716                         ($drive->{mbps_wr} || 0)*1024*1024,
4717                          $drive->{iops} || 0,
4718                          $drive->{iops_rd} || 0,
4719                          $drive->{iops_wr} || 0,
4720                         ($drive->{mbps_max} || 0)*1024*1024,
4721                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4722                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4723                          $drive->{iops_max} || 0,
4724                          $drive->{iops_rd_max} || 0,
4725                          $drive->{iops_wr_max} || 0,
4726                          $drive->{bps_max_length} || 1,
4727                          $drive->{bps_rd_max_length} || 1,
4728                          $drive->{bps_wr_max_length} || 1,
4729                          $drive->{iops_max_length} || 1,
4730                          $drive->{iops_rd_max_length} || 1,
4731                          $drive->{iops_wr_max_length} || 1,
4732                     );
4733
4734                 }
4735
4736                 return 1;
4737             }
4738
4739         } else { # cdrom
4740
4741             if ($drive->{file} eq 'none') {
4742                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4743                 if (drive_is_cloudinit($old_drive)) {
4744                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4745                 }
4746             } else {
4747                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4748
4749                 # force eject if locked
4750                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4751
4752                 if ($path) {
4753                     mon_cmd($vmid, "blockdev-change-medium",
4754                         id => "$opt", filename => "$path");
4755                 }
4756             }
4757
4758             return 1;
4759         }
4760     }
4761
4762     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4763     # hotplug new disks
4764     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4765     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4766 }
4767
4768 # called in locked context by incoming migration
4769 sub vm_migrate_get_nbd_disks {
4770     my ($storecfg, $conf, $replicated_volumes) = @_;
4771
4772     my $local_volumes = {};
4773     PVE::QemuConfig->foreach_volume($conf, sub {
4774         my ($ds, $drive) = @_;
4775
4776         return if drive_is_cdrom($drive);
4777
4778         my $volid = $drive->{file};
4779
4780         return if !$volid;
4781
4782         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4783
4784         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4785         return if $scfg->{shared};
4786
4787         # replicated disks re-use existing state via bitmap
4788         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4789         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4790     });
4791     return $local_volumes;
4792 }
4793
4794 # called in locked context by incoming migration
4795 sub vm_migrate_alloc_nbd_disks {
4796     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4797
4798     my $format = undef;
4799
4800     my $nbd = {};
4801     foreach my $opt (sort keys %$source_volumes) {
4802         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4803
4804         if ($use_existing) {
4805             $nbd->{$opt}->{drivestr} = print_drive($drive);
4806             $nbd->{$opt}->{volid} = $volid;
4807             $nbd->{$opt}->{replicated} = 1;
4808             next;
4809         }
4810
4811         # If a remote storage is specified and the format of the original
4812         # volume is not available there, fall back to the default format.
4813         # Otherwise use the same format as the original.
4814         if (!$storagemap->{identity}) {
4815             $storeid = map_storage($storagemap, $storeid);
4816             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4817             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4818             my $fileFormat = qemu_img_format($scfg, $volname);
4819             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4820         } else {
4821             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4822             $format = qemu_img_format($scfg, $volname);
4823         }
4824
4825         my $size = $drive->{size} / 1024;
4826         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4827         my $newdrive = $drive;
4828         $newdrive->{format} = $format;
4829         $newdrive->{file} = $newvolid;
4830         my $drivestr = print_drive($newdrive);
4831         $nbd->{$opt}->{drivestr} = $drivestr;
4832         $nbd->{$opt}->{volid} = $newvolid;
4833     }
4834
4835     return $nbd;
4836 }
4837
4838 # see vm_start_nolock for parameters, additionally:
4839 # migrate_opts:
4840 #   storagemap = parsed storage map for allocating NBD disks
4841 sub vm_start {
4842     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4843
4844     return PVE::QemuConfig->lock_config($vmid, sub {
4845         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4846
4847         die "you can't start a vm if it's a template\n"
4848             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4849
4850         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4851
4852         PVE::QemuConfig->check_lock($conf)
4853             if !($params->{skiplock} || $has_suspended_lock);
4854
4855         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4856
4857         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4858
4859         if (my $storagemap = $migrate_opts->{storagemap}) {
4860             my $replicated = $migrate_opts->{replicated_volumes};
4861             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4862             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4863
4864             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4865                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4866             }
4867         }
4868
4869         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4870     });
4871 }
4872
4873
4874 # params:
4875 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4876 #   skiplock => 0/1, skip checking for config lock
4877 #   skiptemplate => 0/1, skip checking whether VM is template
4878 #   forcemachine => to force Qemu machine (rollback/migration)
4879 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4880 #   timeout => in seconds
4881 #   paused => start VM in paused state (backup)
4882 #   resume => resume from hibernation
4883 # migrate_opts:
4884 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4885 #   migratedfrom => source node
4886 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4887 #   network => CIDR of migration network
4888 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4889 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4890 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4891 sub vm_start_nolock {
4892     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4893
4894     my $statefile = $params->{statefile};
4895     my $resume = $params->{resume};
4896
4897     my $migratedfrom = $migrate_opts->{migratedfrom};
4898     my $migration_type = $migrate_opts->{type};
4899
4900     my $res = {};
4901
4902     # clean up leftover reboot request files
4903     eval { clear_reboot_request($vmid); };
4904     warn $@ if $@;
4905
4906     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4907         vmconfig_apply_pending($vmid, $conf, $storecfg);
4908         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4909     }
4910
4911     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4912
4913     my $defaults = load_defaults();
4914
4915     # set environment variable useful inside network script
4916     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4917
4918     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4919
4920     my $forcemachine = $params->{forcemachine};
4921     my $forcecpu = $params->{forcecpu};
4922     if ($resume) {
4923         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4924         $forcemachine = $conf->{runningmachine};
4925         $forcecpu = $conf->{runningcpu};
4926         print "Resuming suspended VM\n";
4927     }
4928
4929     my ($cmd, $vollist, $spice_port) =
4930         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4931
4932     my $migration_ip;
4933     my $get_migration_ip = sub {
4934         my ($nodename) = @_;
4935
4936         return $migration_ip if defined($migration_ip);
4937
4938         my $cidr = $migrate_opts->{network};
4939
4940         if (!defined($cidr)) {
4941             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4942             $cidr = $dc_conf->{migration}->{network};
4943         }
4944
4945         if (defined($cidr)) {
4946             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4947
4948             die "could not get IP: no address configured on local " .
4949                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4950
4951             die "could not get IP: multiple addresses configured on local " .
4952                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4953
4954             $migration_ip = @$ips[0];
4955         }
4956
4957         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4958             if !defined($migration_ip);
4959
4960         return $migration_ip;
4961     };
4962
4963     my $migrate_uri;
4964     if ($statefile) {
4965         if ($statefile eq 'tcp') {
4966             my $localip = "localhost";
4967             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4968             my $nodename = nodename();
4969
4970             if (!defined($migration_type)) {
4971                 if (defined($datacenterconf->{migration}->{type})) {
4972                     $migration_type = $datacenterconf->{migration}->{type};
4973                 } else {
4974                     $migration_type = 'secure';
4975                 }
4976             }
4977
4978             if ($migration_type eq 'insecure') {
4979                 $localip = $get_migration_ip->($nodename);
4980                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4981             }
4982
4983             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4984             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4985             $migrate_uri = "tcp:${localip}:${migrate_port}";
4986             push @$cmd, '-incoming', $migrate_uri;
4987             push @$cmd, '-S';
4988
4989         } elsif ($statefile eq 'unix') {
4990             # should be default for secure migrations as a ssh TCP forward
4991             # tunnel is not deterministic reliable ready and fails regurarly
4992             # to set up in time, so use UNIX socket forwards
4993             my $socket_addr = "/run/qemu-server/$vmid.migrate";
4994             unlink $socket_addr;
4995
4996             $migrate_uri = "unix:$socket_addr";
4997
4998             push @$cmd, '-incoming', $migrate_uri;
4999             push @$cmd, '-S';
5000
5001         } elsif (-e $statefile) {
5002             push @$cmd, '-loadstate', $statefile;
5003         } else {
5004             my $statepath = PVE::Storage::path($storecfg, $statefile);
5005             push @$vollist, $statefile;
5006             push @$cmd, '-loadstate', $statepath;
5007         }
5008     } elsif ($params->{paused}) {
5009         push @$cmd, '-S';
5010     }
5011
5012     # host pci devices
5013     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5014       my $d = parse_hostpci($conf->{"hostpci$i"});
5015       next if !$d;
5016       my $pcidevices = $d->{pciid};
5017       foreach my $pcidevice (@$pcidevices) {
5018             my $pciid = $pcidevice->{id};
5019
5020             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5021             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5022             die "no pci device info for device '$pciid'\n" if !$info;
5023
5024             if ($d->{mdev}) {
5025                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5026                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5027             } else {
5028                 die "can't unbind/bind pci group to vfio '$pciid'\n"
5029                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5030                 die "can't reset pci device '$pciid'\n"
5031                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5032             }
5033       }
5034     }
5035
5036     PVE::Storage::activate_volumes($storecfg, $vollist);
5037
5038     eval {
5039         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5040             outfunc => sub {}, errfunc => sub {});
5041     };
5042     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5043     # timeout should be more than enough here...
5044     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5045
5046     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5047                                               : $defaults->{cpuunits};
5048
5049     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5050     my %run_params = (
5051         timeout => $statefile ? undef : $start_timeout,
5052         umask => 0077,
5053         noerr => 1,
5054     );
5055
5056     # when migrating, prefix QEMU output so other side can pick up any
5057     # errors that might occur and show the user
5058     if ($migratedfrom) {
5059         $run_params{quiet} = 1;
5060         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5061     }
5062
5063     my %properties = (
5064         Slice => 'qemu.slice',
5065         KillMode => 'none',
5066         CPUShares => $cpuunits
5067     );
5068
5069     if (my $cpulimit = $conf->{cpulimit}) {
5070         $properties{CPUQuota} = int($cpulimit * 100);
5071     }
5072     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5073
5074     my $run_qemu = sub {
5075         PVE::Tools::run_fork sub {
5076             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5077
5078             my $exitcode = run_command($cmd, %run_params);
5079             die "QEMU exited with code $exitcode\n" if $exitcode;
5080         };
5081     };
5082
5083     if ($conf->{hugepages}) {
5084
5085         my $code = sub {
5086             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5087             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5088
5089             PVE::QemuServer::Memory::hugepages_mount();
5090             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5091
5092             eval { $run_qemu->() };
5093             if (my $err = $@) {
5094                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5095                     if !$conf->{keephugepages};
5096                 die $err;
5097             }
5098
5099             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5100                 if !$conf->{keephugepages};
5101         };
5102         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5103
5104     } else {
5105         eval { $run_qemu->() };
5106     }
5107
5108     if (my $err = $@) {
5109         # deactivate volumes if start fails
5110         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5111         die "start failed: $err";
5112     }
5113
5114     print "migration listens on $migrate_uri\n" if $migrate_uri;
5115     $res->{migrate_uri} = $migrate_uri;
5116
5117     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5118         eval { mon_cmd($vmid, "cont"); };
5119         warn $@ if $@;
5120     }
5121
5122     #start nbd server for storage migration
5123     if (my $nbd = $migrate_opts->{nbd}) {
5124         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5125
5126         my $migrate_storage_uri;
5127         # nbd_protocol_version > 0 for unix socket support
5128         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5129             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5130             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5131             $migrate_storage_uri = "nbd:unix:$socket_path";
5132         } else {
5133             my $nodename = nodename();
5134             my $localip = $get_migration_ip->($nodename);
5135             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5136             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5137
5138             mon_cmd($vmid, "nbd-server-start", addr => {
5139                 type => 'inet',
5140                 data => {
5141                     host => "${localip}",
5142                     port => "${storage_migrate_port}",
5143                 },
5144             });
5145             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5146             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5147         }
5148
5149         $res->{migrate_storage_uri} = $migrate_storage_uri;
5150
5151         foreach my $opt (sort keys %$nbd) {
5152             my $drivestr = $nbd->{$opt}->{drivestr};
5153             my $volid = $nbd->{$opt}->{volid};
5154             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5155             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5156             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5157             print "re-using replicated volume: $opt - $volid\n"
5158                 if $nbd->{$opt}->{replicated};
5159
5160             $res->{drives}->{$opt} = $nbd->{$opt};
5161             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5162         }
5163     }
5164
5165     if ($migratedfrom) {
5166         eval {
5167             set_migration_caps($vmid);
5168         };
5169         warn $@ if $@;
5170
5171         if ($spice_port) {
5172             print "spice listens on port $spice_port\n";
5173             $res->{spice_port} = $spice_port;
5174             if ($migrate_opts->{spice_ticket}) {
5175                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5176                     $migrate_opts->{spice_ticket});
5177                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5178             }
5179         }
5180
5181     } else {
5182         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5183             if !$statefile && $conf->{balloon};
5184
5185         foreach my $opt (keys %$conf) {
5186             next if $opt !~  m/^net\d+$/;
5187             my $nicconf = parse_net($conf->{$opt});
5188             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5189         }
5190     }
5191
5192     mon_cmd($vmid, 'qom-set',
5193                 path => "machine/peripheral/balloon0",
5194                 property => "guest-stats-polling-interval",
5195                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5196
5197     if ($resume) {
5198         print "Resumed VM, removing state\n";
5199         if (my $vmstate = $conf->{vmstate}) {
5200             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5201             PVE::Storage::vdisk_free($storecfg, $vmstate);
5202         }
5203         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5204         PVE::QemuConfig->write_config($vmid, $conf);
5205     }
5206
5207     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5208
5209     return $res;
5210 }
5211
5212 sub vm_commandline {
5213     my ($storecfg, $vmid, $snapname) = @_;
5214
5215     my $conf = PVE::QemuConfig->load_config($vmid);
5216     my $forcemachine;
5217     my $forcecpu;
5218
5219     if ($snapname) {
5220         my $snapshot = $conf->{snapshots}->{$snapname};
5221         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5222
5223         # check for machine or CPU overrides in snapshot
5224         $forcemachine = $snapshot->{runningmachine};
5225         $forcecpu = $snapshot->{runningcpu};
5226
5227         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5228
5229         $conf = $snapshot;
5230     }
5231
5232     my $defaults = load_defaults();
5233
5234     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5235         $forcemachine, $forcecpu);
5236
5237     return PVE::Tools::cmd2string($cmd);
5238 }
5239
5240 sub vm_reset {
5241     my ($vmid, $skiplock) = @_;
5242
5243     PVE::QemuConfig->lock_config($vmid, sub {
5244
5245         my $conf = PVE::QemuConfig->load_config($vmid);
5246
5247         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5248
5249         mon_cmd($vmid, "system_reset");
5250     });
5251 }
5252
5253 sub get_vm_volumes {
5254     my ($conf) = @_;
5255
5256     my $vollist = [];
5257     foreach_volid($conf, sub {
5258         my ($volid, $attr) = @_;
5259
5260         return if $volid =~ m|^/|;
5261
5262         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5263         return if !$sid;
5264
5265         push @$vollist, $volid;
5266     });
5267
5268     return $vollist;
5269 }
5270
5271 sub vm_stop_cleanup {
5272     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5273
5274     eval {
5275
5276         if (!$keepActive) {
5277             my $vollist = get_vm_volumes($conf);
5278             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5279         }
5280
5281         foreach my $ext (qw(mon qmp pid vnc qga)) {
5282             unlink "/var/run/qemu-server/${vmid}.$ext";
5283         }
5284
5285         if ($conf->{ivshmem}) {
5286             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5287             # just delete it for now, VMs which have this already open do not
5288             # are affected, but new VMs will get a separated one. If this
5289             # becomes an issue we either add some sort of ref-counting or just
5290             # add a "don't delete on stop" flag to the ivshmem format.
5291             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5292         }
5293
5294         foreach my $key (keys %$conf) {
5295             next if $key !~ m/^hostpci(\d+)$/;
5296             my $hostpciindex = $1;
5297             my $d = parse_hostpci($conf->{$key});
5298             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5299
5300             foreach my $pci (@{$d->{pciid}}) {
5301                 my $pciid = $pci->{id};
5302                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5303             }
5304         }
5305
5306         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5307     };
5308     warn $@ if $@; # avoid errors - just warn
5309 }
5310
5311 # call only in locked context
5312 sub _do_vm_stop {
5313     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5314
5315     my $pid = check_running($vmid, $nocheck);
5316     return if !$pid;
5317
5318     my $conf;
5319     if (!$nocheck) {
5320         $conf = PVE::QemuConfig->load_config($vmid);
5321         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5322         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5323             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5324             $timeout = $opts->{down} if $opts->{down};
5325         }
5326         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5327     }
5328
5329     eval {
5330         if ($shutdown) {
5331             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5332                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5333             } else {
5334                 mon_cmd($vmid, "system_powerdown");
5335             }
5336         } else {
5337             mon_cmd($vmid, "quit");
5338         }
5339     };
5340     my $err = $@;
5341
5342     if (!$err) {
5343         $timeout = 60 if !defined($timeout);
5344
5345         my $count = 0;
5346         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5347             $count++;
5348             sleep 1;
5349         }
5350
5351         if ($count >= $timeout) {
5352             if ($force) {
5353                 warn "VM still running - terminating now with SIGTERM\n";
5354                 kill 15, $pid;
5355             } else {
5356                 die "VM quit/powerdown failed - got timeout\n";
5357             }
5358         } else {
5359             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5360             return;
5361         }
5362     } else {
5363        if (!check_running($vmid, $nocheck)) {
5364             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5365             return;
5366        }
5367        if ($force) {
5368             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5369             kill 15, $pid;
5370         } else {
5371             die "VM quit/powerdown failed\n";
5372         }
5373     }
5374
5375     # wait again
5376     $timeout = 10;
5377
5378     my $count = 0;
5379     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5380         $count++;
5381         sleep 1;
5382     }
5383
5384     if ($count >= $timeout) {
5385         warn "VM still running - terminating now with SIGKILL\n";
5386         kill 9, $pid;
5387         sleep 1;
5388     }
5389
5390     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5391 }
5392
5393 # Note: use $nocheck to skip tests if VM configuration file exists.
5394 # We need that when migration VMs to other nodes (files already moved)
5395 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5396 sub vm_stop {
5397     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5398
5399     $force = 1 if !defined($force) && !$shutdown;
5400
5401     if ($migratedfrom){
5402         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5403         kill 15, $pid if $pid;
5404         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5405         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5406         return;
5407     }
5408
5409     PVE::QemuConfig->lock_config($vmid, sub {
5410         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5411    });
5412 }
5413
5414 sub vm_reboot {
5415     my ($vmid, $timeout) = @_;
5416
5417     PVE::QemuConfig->lock_config($vmid, sub {
5418         eval {
5419
5420             # only reboot if running, as qmeventd starts it again on a stop event
5421             return if !check_running($vmid);
5422
5423             create_reboot_request($vmid);
5424
5425             my $storecfg = PVE::Storage::config();
5426             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5427
5428         };
5429         if (my $err = $@) {
5430             # avoid that the next normal shutdown will be confused for a reboot
5431             clear_reboot_request($vmid);
5432             die $err;
5433         }
5434    });
5435 }
5436
5437 # note: if using the statestorage parameter, the caller has to check privileges
5438 sub vm_suspend {
5439     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5440
5441     my $conf;
5442     my $path;
5443     my $storecfg;
5444     my $vmstate;
5445
5446     PVE::QemuConfig->lock_config($vmid, sub {
5447
5448         $conf = PVE::QemuConfig->load_config($vmid);
5449
5450         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5451         PVE::QemuConfig->check_lock($conf)
5452             if !($skiplock || $is_backing_up);
5453
5454         die "cannot suspend to disk during backup\n"
5455             if $is_backing_up && $includestate;
5456
5457         if ($includestate) {
5458             $conf->{lock} = 'suspending';
5459             my $date = strftime("%Y-%m-%d", localtime(time()));
5460             $storecfg = PVE::Storage::config();
5461             if (!$statestorage) {
5462                 $statestorage = find_vmstate_storage($conf, $storecfg);
5463                 # check permissions for the storage
5464                 my $rpcenv = PVE::RPCEnvironment::get();
5465                 if ($rpcenv->{type} ne 'cli') {
5466                     my $authuser = $rpcenv->get_user();
5467                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5468                 }
5469             }
5470
5471
5472             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5473                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5474             $path = PVE::Storage::path($storecfg, $vmstate);
5475             PVE::QemuConfig->write_config($vmid, $conf);
5476         } else {
5477             mon_cmd($vmid, "stop");
5478         }
5479     });
5480
5481     if ($includestate) {
5482         # save vm state
5483         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5484
5485         eval {
5486             mon_cmd($vmid, "savevm-start", statefile => $path);
5487             for(;;) {
5488                 my $state = mon_cmd($vmid, "query-savevm");
5489                 if (!$state->{status}) {
5490                     die "savevm not active\n";
5491                 } elsif ($state->{status} eq 'active') {
5492                     sleep(1);
5493                     next;
5494                 } elsif ($state->{status} eq 'completed') {
5495                     print "State saved, quitting\n";
5496                     last;
5497                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5498                     die "query-savevm failed with error '$state->{error}'\n"
5499                 } else {
5500                     die "query-savevm returned status '$state->{status}'\n";
5501                 }
5502             }
5503         };
5504         my $err = $@;
5505
5506         PVE::QemuConfig->lock_config($vmid, sub {
5507             $conf = PVE::QemuConfig->load_config($vmid);
5508             if ($err) {
5509                 # cleanup, but leave suspending lock, to indicate something went wrong
5510                 eval {
5511                     mon_cmd($vmid, "savevm-end");
5512                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5513                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5514                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5515                     PVE::QemuConfig->write_config($vmid, $conf);
5516                 };
5517                 warn $@ if $@;
5518                 die $err;
5519             }
5520
5521             die "lock changed unexpectedly\n"
5522                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5523
5524             mon_cmd($vmid, "quit");
5525             $conf->{lock} = 'suspended';
5526             PVE::QemuConfig->write_config($vmid, $conf);
5527         });
5528     }
5529 }
5530
5531 sub vm_resume {
5532     my ($vmid, $skiplock, $nocheck) = @_;
5533
5534     PVE::QemuConfig->lock_config($vmid, sub {
5535         my $res = mon_cmd($vmid, 'query-status');
5536         my $resume_cmd = 'cont';
5537
5538         if ($res->{status} && $res->{status} eq 'suspended') {
5539             $resume_cmd = 'system_wakeup';
5540         }
5541
5542         if (!$nocheck) {
5543
5544             my $conf = PVE::QemuConfig->load_config($vmid);
5545
5546             PVE::QemuConfig->check_lock($conf)
5547                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5548         }
5549
5550         mon_cmd($vmid, $resume_cmd);
5551     });
5552 }
5553
5554 sub vm_sendkey {
5555     my ($vmid, $skiplock, $key) = @_;
5556
5557     PVE::QemuConfig->lock_config($vmid, sub {
5558
5559         my $conf = PVE::QemuConfig->load_config($vmid);
5560
5561         # there is no qmp command, so we use the human monitor command
5562         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5563         die $res if $res ne '';
5564     });
5565 }
5566
5567 # vzdump restore implementaion
5568
5569 sub tar_archive_read_firstfile {
5570     my $archive = shift;
5571
5572     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5573
5574     # try to detect archive type first
5575     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5576         die "unable to open file '$archive'\n";
5577     my $firstfile = <$fh>;
5578     kill 15, $pid;
5579     close $fh;
5580
5581     die "ERROR: archive contaions no data\n" if !$firstfile;
5582     chomp $firstfile;
5583
5584     return $firstfile;
5585 }
5586
5587 sub tar_restore_cleanup {
5588     my ($storecfg, $statfile) = @_;
5589
5590     print STDERR "starting cleanup\n";
5591
5592     if (my $fd = IO::File->new($statfile, "r")) {
5593         while (defined(my $line = <$fd>)) {
5594             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5595                 my $volid = $2;
5596                 eval {
5597                     if ($volid =~ m|^/|) {
5598                         unlink $volid || die 'unlink failed\n';
5599                     } else {
5600                         PVE::Storage::vdisk_free($storecfg, $volid);
5601                     }
5602                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5603                 };
5604                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5605             } else {
5606                 print STDERR "unable to parse line in statfile - $line";
5607             }
5608         }
5609         $fd->close();
5610     }
5611 }
5612
5613 sub restore_file_archive {
5614     my ($archive, $vmid, $user, $opts) = @_;
5615
5616     return restore_vma_archive($archive, $vmid, $user, $opts)
5617         if $archive eq '-';
5618
5619     my $info = PVE::Storage::archive_info($archive);
5620     my $format = $opts->{format} // $info->{format};
5621     my $comp = $info->{compression};
5622
5623     # try to detect archive format
5624     if ($format eq 'tar') {
5625         return restore_tar_archive($archive, $vmid, $user, $opts);
5626     } else {
5627         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5628     }
5629 }
5630
5631 # hepler to remove disks that will not be used after restore
5632 my $restore_cleanup_oldconf = sub {
5633     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5634
5635     PVE::QemuConfig->foreach_volume($oldconf, sub {
5636         my ($ds, $drive) = @_;
5637
5638         return if drive_is_cdrom($drive, 1);
5639
5640         my $volid = $drive->{file};
5641         return if !$volid || $volid =~ m|^/|;
5642
5643         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5644         return if !$path || !$owner || ($owner != $vmid);
5645
5646         # Note: only delete disk we want to restore
5647         # other volumes will become unused
5648         if ($virtdev_hash->{$ds}) {
5649             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5650             if (my $err = $@) {
5651                 warn $err;
5652             }
5653         }
5654     });
5655
5656     # delete vmstate files, after the restore we have no snapshots anymore
5657     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5658         my $snap = $oldconf->{snapshots}->{$snapname};
5659         if ($snap->{vmstate}) {
5660             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5661             if (my $err = $@) {
5662                 warn $err;
5663             }
5664         }
5665     }
5666 };
5667
5668 # Helper to parse vzdump backup device hints
5669 #
5670 # $rpcenv: Environment, used to ckeck storage permissions
5671 # $user: User ID, to check storage permissions
5672 # $storecfg: Storage configuration
5673 # $fh: the file handle for reading the configuration
5674 # $devinfo: should contain device sizes for all backu-up'ed devices
5675 # $options: backup options (pool, default storage)
5676 #
5677 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5678 my $parse_backup_hints = sub {
5679     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5680
5681     my $virtdev_hash = {};
5682
5683     while (defined(my $line = <$fh>)) {
5684         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5685             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5686             die "archive does not contain data for drive '$virtdev'\n"
5687                 if !$devinfo->{$devname};
5688
5689             if (defined($options->{storage})) {
5690                 $storeid = $options->{storage} || 'local';
5691             } elsif (!$storeid) {
5692                 $storeid = 'local';
5693             }
5694             $format = 'raw' if !$format;
5695             $devinfo->{$devname}->{devname} = $devname;
5696             $devinfo->{$devname}->{virtdev} = $virtdev;
5697             $devinfo->{$devname}->{format} = $format;
5698             $devinfo->{$devname}->{storeid} = $storeid;
5699
5700             # check permission on storage
5701             my $pool = $options->{pool}; # todo: do we need that?
5702             if ($user ne 'root@pam') {
5703                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5704             }
5705
5706             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5707         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5708             my $virtdev = $1;
5709             my $drive = parse_drive($virtdev, $2);
5710             if (drive_is_cloudinit($drive)) {
5711                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5712                 $storeid = $options->{storage} if defined ($options->{storage});
5713                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5714                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5715
5716                 $virtdev_hash->{$virtdev} = {
5717                     format => $format,
5718                     storeid => $storeid,
5719                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5720                     is_cloudinit => 1,
5721                 };
5722             }
5723         }
5724     }
5725
5726     return $virtdev_hash;
5727 };
5728
5729 # Helper to allocate and activate all volumes required for a restore
5730 #
5731 # $storecfg: Storage configuration
5732 # $virtdev_hash: as returned by parse_backup_hints()
5733 #
5734 # Returns: { $virtdev => $volid }
5735 my $restore_allocate_devices = sub {
5736     my ($storecfg, $virtdev_hash, $vmid) = @_;
5737
5738     my $map = {};
5739     foreach my $virtdev (sort keys %$virtdev_hash) {
5740         my $d = $virtdev_hash->{$virtdev};
5741         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5742         my $storeid = $d->{storeid};
5743         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5744
5745         # test if requested format is supported
5746         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5747         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5748         $d->{format} = $defFormat if !$supported;
5749
5750         my $name;
5751         if ($d->{is_cloudinit}) {
5752             $name = "vm-$vmid-cloudinit";
5753             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5754         }
5755
5756         my $volid = PVE::Storage::vdisk_alloc(
5757             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5758
5759         print STDERR "new volume ID is '$volid'\n";
5760         $d->{volid} = $volid;
5761
5762         PVE::Storage::activate_volumes($storecfg, [$volid]);
5763
5764         $map->{$virtdev} = $volid;
5765     }
5766
5767     return $map;
5768 };
5769
5770 my $restore_update_config_line = sub {
5771     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5772
5773     return if $line =~ m/^\#qmdump\#/;
5774     return if $line =~ m/^\#vzdump\#/;
5775     return if $line =~ m/^lock:/;
5776     return if $line =~ m/^unused\d+:/;
5777     return if $line =~ m/^parent:/;
5778
5779     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5780     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5781         # try to convert old 1.X settings
5782         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5783         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5784             my ($model, $macaddr) = split(/\=/, $devconfig);
5785             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5786             my $net = {
5787                 model => $model,
5788                 bridge => "vmbr$ind",
5789                 macaddr => $macaddr,
5790             };
5791             my $netstr = print_net($net);
5792
5793             print $outfd "net$cookie->{netcount}: $netstr\n";
5794             $cookie->{netcount}++;
5795         }
5796     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5797         my ($id, $netstr) = ($1, $2);
5798         my $net = parse_net($netstr);
5799         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5800         $netstr = print_net($net);
5801         print $outfd "$id: $netstr\n";
5802     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5803         my $virtdev = $1;
5804         my $value = $3;
5805         my $di = parse_drive($virtdev, $value);
5806         if (defined($di->{backup}) && !$di->{backup}) {
5807             print $outfd "#$line";
5808         } elsif ($map->{$virtdev}) {
5809             delete $di->{format}; # format can change on restore
5810             $di->{file} = $map->{$virtdev};
5811             $value = print_drive($di);
5812             print $outfd "$virtdev: $value\n";
5813         } else {
5814             print $outfd $line;
5815         }
5816     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5817         my $vmgenid = $1;
5818         if ($vmgenid ne '0') {
5819             # always generate a new vmgenid if there was a valid one setup
5820             $vmgenid = generate_uuid();
5821         }
5822         print $outfd "vmgenid: $vmgenid\n";
5823     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5824         my ($uuid, $uuid_str);
5825         UUID::generate($uuid);
5826         UUID::unparse($uuid, $uuid_str);
5827         my $smbios1 = parse_smbios1($2);
5828         $smbios1->{uuid} = $uuid_str;
5829         print $outfd $1.print_smbios1($smbios1)."\n";
5830     } else {
5831         print $outfd $line;
5832     }
5833 };
5834
5835 my $restore_deactivate_volumes = sub {
5836     my ($storecfg, $devinfo) = @_;
5837
5838     my $vollist = [];
5839     foreach my $devname (keys %$devinfo) {
5840         my $volid = $devinfo->{$devname}->{volid};
5841         push @$vollist, $volid if $volid;
5842     }
5843
5844     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5845 };
5846
5847 my $restore_destroy_volumes = sub {
5848     my ($storecfg, $devinfo) = @_;
5849
5850     foreach my $devname (keys %$devinfo) {
5851         my $volid = $devinfo->{$devname}->{volid};
5852         next if !$volid;
5853         eval {
5854             if ($volid =~ m|^/|) {
5855                 unlink $volid || die 'unlink failed\n';
5856             } else {
5857                 PVE::Storage::vdisk_free($storecfg, $volid);
5858             }
5859             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5860         };
5861         print STDERR "unable to cleanup '$volid' - $@" if $@;
5862     }
5863 };
5864
5865 sub scan_volids {
5866     my ($cfg, $vmid) = @_;
5867
5868     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5869
5870     my $volid_hash = {};
5871     foreach my $storeid (keys %$info) {
5872         foreach my $item (@{$info->{$storeid}}) {
5873             next if !($item->{volid} && $item->{size});
5874             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5875             $volid_hash->{$item->{volid}} = $item;
5876         }
5877     }
5878
5879     return $volid_hash;
5880 }
5881
5882 sub update_disk_config {
5883     my ($vmid, $conf, $volid_hash) = @_;
5884
5885     my $changes;
5886     my $prefix = "VM $vmid";
5887
5888     # used and unused disks
5889     my $referenced = {};
5890
5891     # Note: it is allowed to define multiple storages with same path (alias), so
5892     # we need to check both 'volid' and real 'path' (two different volid can point
5893     # to the same path).
5894
5895     my $referencedpath = {};
5896
5897     # update size info
5898     PVE::QemuConfig->foreach_volume($conf, sub {
5899         my ($opt, $drive) = @_;
5900
5901         my $volid = $drive->{file};
5902         return if !$volid;
5903         my $volume = $volid_hash->{$volid};
5904
5905         # mark volid as "in-use" for next step
5906         $referenced->{$volid} = 1;
5907         if ($volume && (my $path = $volume->{path})) {
5908             $referencedpath->{$path} = 1;
5909         }
5910
5911         return if drive_is_cdrom($drive);
5912         return if !$volume;
5913
5914         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5915         if (defined($updated)) {
5916             $changes = 1;
5917             $conf->{$opt} = print_drive($updated);
5918             print "$prefix ($opt): $msg\n";
5919         }
5920     });
5921
5922     # remove 'unusedX' entry if volume is used
5923     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5924         my ($opt, $drive) = @_;
5925
5926         my $volid = $drive->{file};
5927         return if !$volid;
5928
5929         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5930         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5931             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5932             $changes = 1;
5933             delete $conf->{$opt};
5934         }
5935
5936         $referenced->{$volid} = 1;
5937         $referencedpath->{$path} = 1 if $path;
5938     });
5939
5940     foreach my $volid (sort keys %$volid_hash) {
5941         next if $volid =~ m/vm-$vmid-state-/;
5942         next if $referenced->{$volid};
5943         my $path = $volid_hash->{$volid}->{path};
5944         next if !$path; # just to be sure
5945         next if $referencedpath->{$path};
5946         $changes = 1;
5947         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5948         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5949         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5950     }
5951
5952     return $changes;
5953 }
5954
5955 sub rescan {
5956     my ($vmid, $nolock, $dryrun) = @_;
5957
5958     my $cfg = PVE::Storage::config();
5959
5960     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5961     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5962     foreach my $stor (keys %{$cfg->{ids}}) {
5963         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5964     }
5965
5966     print "rescan volumes...\n";
5967     my $volid_hash = scan_volids($cfg, $vmid);
5968
5969     my $updatefn =  sub {
5970         my ($vmid) = @_;
5971
5972         my $conf = PVE::QemuConfig->load_config($vmid);
5973
5974         PVE::QemuConfig->check_lock($conf);
5975
5976         my $vm_volids = {};
5977         foreach my $volid (keys %$volid_hash) {
5978             my $info = $volid_hash->{$volid};
5979             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5980         }
5981
5982         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5983
5984         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5985     };
5986
5987     if (defined($vmid)) {
5988         if ($nolock) {
5989             &$updatefn($vmid);
5990         } else {
5991             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5992         }
5993     } else {
5994         my $vmlist = config_list();
5995         foreach my $vmid (keys %$vmlist) {
5996             if ($nolock) {
5997                 &$updatefn($vmid);
5998             } else {
5999                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6000             }
6001         }
6002     }
6003 }
6004
6005 sub restore_proxmox_backup_archive {
6006     my ($archive, $vmid, $user, $options) = @_;
6007
6008     my $storecfg = PVE::Storage::config();
6009
6010     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6011     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6012
6013     my $server = $scfg->{server};
6014     my $datastore = $scfg->{datastore};
6015     my $username = $scfg->{username} // 'root@pam';
6016     my $fingerprint = $scfg->{fingerprint};
6017     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6018
6019     my $repo = "$username\@$server:$datastore";
6020
6021     # This is only used for `pbs-restore`!
6022     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6023     local $ENV{PBS_PASSWORD} = $password;
6024     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6025
6026     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6027         PVE::Storage::parse_volname($storecfg, $archive);
6028
6029     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6030
6031     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6032
6033     my $tmpdir = "/var/tmp/vzdumptmp$$";
6034     rmtree $tmpdir;
6035     mkpath $tmpdir;
6036
6037     my $conffile = PVE::QemuConfig->config_file($vmid);
6038     my $tmpfn = "$conffile.$$.tmp";
6039      # disable interrupts (always do cleanups)
6040     local $SIG{INT} =
6041         local $SIG{TERM} =
6042         local $SIG{QUIT} =
6043         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6044
6045     # Note: $oldconf is undef if VM does not exists
6046     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6047     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6048
6049     my $rpcenv = PVE::RPCEnvironment::get();
6050     my $devinfo = {};
6051
6052     eval {
6053         # enable interrupts
6054         local $SIG{INT} =
6055             local $SIG{TERM} =
6056             local $SIG{QUIT} =
6057             local $SIG{HUP} =
6058             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6059
6060         my $cfgfn = "$tmpdir/qemu-server.conf";
6061         my $firewall_config_fn = "$tmpdir/fw.conf";
6062         my $index_fn = "$tmpdir/index.json";
6063
6064         my $cmd = "restore";
6065
6066         my $param = [$pbs_backup_name, "index.json", $index_fn];
6067         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6068         my $index = PVE::Tools::file_get_contents($index_fn);
6069         $index = decode_json($index);
6070
6071         # print Dumper($index);
6072         foreach my $info (@{$index->{files}}) {
6073             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6074                 my $devname = $1;
6075                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6076                     $devinfo->{$devname}->{size} = $1;
6077                 } else {
6078                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6079                 }
6080             }
6081         }
6082
6083         my $is_qemu_server_backup = scalar(
6084             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6085         );
6086         if (!$is_qemu_server_backup) {
6087             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6088         }
6089         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6090
6091         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6092         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6093
6094         if ($has_firewall_config) {
6095             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6096             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6097
6098             my $pve_firewall_dir = '/etc/pve/firewall';
6099             mkdir $pve_firewall_dir; # make sure the dir exists
6100             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6101         }
6102
6103         my $fh = IO::File->new($cfgfn, "r") ||
6104             die "unable to read qemu-server.conf - $!\n";
6105
6106         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6107
6108         # fixme: rate limit?
6109
6110         # create empty/temp config
6111         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6112
6113         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6114
6115         # allocate volumes
6116         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6117
6118         foreach my $virtdev (sort keys %$virtdev_hash) {
6119             my $d = $virtdev_hash->{$virtdev};
6120             next if $d->{is_cloudinit}; # no need to restore cloudinit
6121
6122             my $volid = $d->{volid};
6123
6124             my $path = PVE::Storage::path($storecfg, $volid);
6125
6126             # This is the ONLY user of the PBS_ env vars set on top of this function!
6127             my $pbs_restore_cmd = [
6128                 '/usr/bin/pbs-restore',
6129                 '--repository', $repo,
6130                 $pbs_backup_name,
6131                 "$d->{devname}.img.fidx",
6132                 $path,
6133                 '--verbose',
6134                 ];
6135
6136             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6137             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6138
6139             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6140                 push @$pbs_restore_cmd, '--skip-zero';
6141             }
6142
6143             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6144             print "restore proxmox backup image: $dbg_cmdstring\n";
6145             run_command($pbs_restore_cmd);
6146         }
6147
6148         $fh->seek(0, 0) || die "seek failed - $!\n";
6149
6150         my $outfd = new IO::File ($tmpfn, "w") ||
6151             die "unable to write config for VM $vmid\n";
6152
6153         my $cookie = { netcount => 0 };
6154         while (defined(my $line = <$fh>)) {
6155             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6156         }
6157
6158         $fh->close();
6159         $outfd->close();
6160     };
6161     my $err = $@;
6162
6163     $restore_deactivate_volumes->($storecfg, $devinfo);
6164
6165     rmtree $tmpdir;
6166
6167     if ($err) {
6168         unlink $tmpfn;
6169         $restore_destroy_volumes->($storecfg, $devinfo);
6170         die $err;
6171     }
6172
6173     rename($tmpfn, $conffile) ||
6174         die "unable to commit configuration file '$conffile'\n";
6175
6176     PVE::Cluster::cfs_update(); # make sure we read new file
6177
6178     eval { rescan($vmid, 1); };
6179     warn $@ if $@;
6180 }
6181
6182 sub restore_vma_archive {
6183     my ($archive, $vmid, $user, $opts, $comp) = @_;
6184
6185     my $readfrom = $archive;
6186
6187     my $cfg = PVE::Storage::config();
6188     my $commands = [];
6189     my $bwlimit = $opts->{bwlimit};
6190
6191     my $dbg_cmdstring = '';
6192     my $add_pipe = sub {
6193         my ($cmd) = @_;
6194         push @$commands, $cmd;
6195         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6196         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6197         $readfrom = '-';
6198     };
6199
6200     my $input = undef;
6201     if ($archive eq '-') {
6202         $input = '<&STDIN';
6203     } else {
6204         # If we use a backup from a PVE defined storage we also consider that
6205         # storage's rate limit:
6206         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6207         if (defined($volid)) {
6208             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6209             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6210             if ($readlimit) {
6211                 print STDERR "applying read rate limit: $readlimit\n";
6212                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6213                 $add_pipe->($cstream);
6214             }
6215         }
6216     }
6217
6218     if ($comp) {
6219         my $info = PVE::Storage::decompressor_info('vma', $comp);
6220         my $cmd = $info->{decompressor};
6221         push @$cmd, $readfrom;
6222         $add_pipe->($cmd);
6223     }
6224
6225     my $tmpdir = "/var/tmp/vzdumptmp$$";
6226     rmtree $tmpdir;
6227
6228     # disable interrupts (always do cleanups)
6229     local $SIG{INT} =
6230         local $SIG{TERM} =
6231         local $SIG{QUIT} =
6232         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6233
6234     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6235     POSIX::mkfifo($mapfifo, 0600);
6236     my $fifofh;
6237
6238     my $openfifo = sub {
6239         open($fifofh, '>', $mapfifo) || die $!;
6240     };
6241
6242     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6243
6244     my $oldtimeout;
6245     my $timeout = 5;
6246
6247     my $devinfo = {};
6248
6249     my $rpcenv = PVE::RPCEnvironment::get();
6250
6251     my $conffile = PVE::QemuConfig->config_file($vmid);
6252     my $tmpfn = "$conffile.$$.tmp";
6253
6254     # Note: $oldconf is undef if VM does not exist
6255     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6256     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6257
6258     my %storage_limits;
6259
6260     my $print_devmap = sub {
6261         my $cfgfn = "$tmpdir/qemu-server.conf";
6262
6263         # we can read the config - that is already extracted
6264         my $fh = IO::File->new($cfgfn, "r") ||
6265             die "unable to read qemu-server.conf - $!\n";
6266
6267         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6268         if (-f $fwcfgfn) {
6269             my $pve_firewall_dir = '/etc/pve/firewall';
6270             mkdir $pve_firewall_dir; # make sure the dir exists
6271             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6272         }
6273
6274         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6275
6276         foreach my $key (keys %storage_limits) {
6277             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6278             next if !$limit;
6279             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6280             $storage_limits{$key} = $limit * 1024;
6281         }
6282
6283         foreach my $devname (keys %$devinfo) {
6284             die "found no device mapping information for device '$devname'\n"
6285                 if !$devinfo->{$devname}->{virtdev};
6286         }
6287
6288         # create empty/temp config
6289         if ($oldconf) {
6290             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6291             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6292         }
6293
6294         # allocate volumes
6295         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6296
6297         # print restore information to $fifofh
6298         foreach my $virtdev (sort keys %$virtdev_hash) {
6299             my $d = $virtdev_hash->{$virtdev};
6300             next if $d->{is_cloudinit}; # no need to restore cloudinit
6301
6302             my $storeid = $d->{storeid};
6303             my $volid = $d->{volid};
6304
6305             my $map_opts = '';
6306             if (my $limit = $storage_limits{$storeid}) {
6307                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6308             }
6309
6310             my $write_zeros = 1;
6311             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6312                 $write_zeros = 0;
6313             }
6314
6315             my $path = PVE::Storage::path($cfg, $volid);
6316
6317             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6318
6319             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6320         }
6321
6322         $fh->seek(0, 0) || die "seek failed - $!\n";
6323
6324         my $outfd = new IO::File ($tmpfn, "w") ||
6325             die "unable to write config for VM $vmid\n";
6326
6327         my $cookie = { netcount => 0 };
6328         while (defined(my $line = <$fh>)) {
6329             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6330         }
6331
6332         $fh->close();
6333         $outfd->close();
6334     };
6335
6336     eval {
6337         # enable interrupts
6338         local $SIG{INT} =
6339             local $SIG{TERM} =
6340             local $SIG{QUIT} =
6341             local $SIG{HUP} =
6342             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6343         local $SIG{ALRM} = sub { die "got timeout\n"; };
6344
6345         $oldtimeout = alarm($timeout);
6346
6347         my $parser = sub {
6348             my $line = shift;
6349
6350             print "$line\n";
6351
6352             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6353                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6354                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6355             } elsif ($line =~ m/^CTIME: /) {
6356                 # we correctly received the vma config, so we can disable
6357                 # the timeout now for disk allocation (set to 10 minutes, so
6358                 # that we always timeout if something goes wrong)
6359                 alarm(600);
6360                 &$print_devmap();
6361                 print $fifofh "done\n";
6362                 my $tmp = $oldtimeout || 0;
6363                 $oldtimeout = undef;
6364                 alarm($tmp);
6365                 close($fifofh);
6366             }
6367         };
6368
6369         print "restore vma archive: $dbg_cmdstring\n";
6370         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6371     };
6372     my $err = $@;
6373
6374     alarm($oldtimeout) if $oldtimeout;
6375
6376     $restore_deactivate_volumes->($cfg, $devinfo);
6377
6378     unlink $mapfifo;
6379     rmtree $tmpdir;
6380
6381     if ($err) {
6382         unlink $tmpfn;
6383         $restore_destroy_volumes->($cfg, $devinfo);
6384         die $err;
6385     }
6386
6387     rename($tmpfn, $conffile) ||
6388         die "unable to commit configuration file '$conffile'\n";
6389
6390     PVE::Cluster::cfs_update(); # make sure we read new file
6391
6392     eval { rescan($vmid, 1); };
6393     warn $@ if $@;
6394 }
6395
6396 sub restore_tar_archive {
6397     my ($archive, $vmid, $user, $opts) = @_;
6398
6399     if ($archive ne '-') {
6400         my $firstfile = tar_archive_read_firstfile($archive);
6401         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6402             if $firstfile ne 'qemu-server.conf';
6403     }
6404
6405     my $storecfg = PVE::Storage::config();
6406
6407     # avoid zombie disks when restoring over an existing VM -> cleanup first
6408     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6409     # skiplock=1 because qmrestore has set the 'create' lock itself already
6410     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6411     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6412
6413     my $tocmd = "/usr/lib/qemu-server/qmextract";
6414
6415     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6416     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6417     $tocmd .= ' --prealloc' if $opts->{prealloc};
6418     $tocmd .= ' --info' if $opts->{info};
6419
6420     # tar option "xf" does not autodetect compression when read from STDIN,
6421     # so we pipe to zcat
6422     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6423         PVE::Tools::shellquote("--to-command=$tocmd");
6424
6425     my $tmpdir = "/var/tmp/vzdumptmp$$";
6426     mkpath $tmpdir;
6427
6428     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6429     local $ENV{VZDUMP_VMID} = $vmid;
6430     local $ENV{VZDUMP_USER} = $user;
6431
6432     my $conffile = PVE::QemuConfig->config_file($vmid);
6433     my $tmpfn = "$conffile.$$.tmp";
6434
6435     # disable interrupts (always do cleanups)
6436     local $SIG{INT} =
6437         local $SIG{TERM} =
6438         local $SIG{QUIT} =
6439         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6440
6441     eval {
6442         # enable interrupts
6443         local $SIG{INT} =
6444             local $SIG{TERM} =
6445             local $SIG{QUIT} =
6446             local $SIG{HUP} =
6447             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6448
6449         if ($archive eq '-') {
6450             print "extracting archive from STDIN\n";
6451             run_command($cmd, input => "<&STDIN");
6452         } else {
6453             print "extracting archive '$archive'\n";
6454             run_command($cmd);
6455         }
6456
6457         return if $opts->{info};
6458
6459         # read new mapping
6460         my $map = {};
6461         my $statfile = "$tmpdir/qmrestore.stat";
6462         if (my $fd = IO::File->new($statfile, "r")) {
6463             while (defined (my $line = <$fd>)) {
6464                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6465                     $map->{$1} = $2 if $1;
6466                 } else {
6467                     print STDERR "unable to parse line in statfile - $line\n";
6468                 }
6469             }
6470             $fd->close();
6471         }
6472
6473         my $confsrc = "$tmpdir/qemu-server.conf";
6474
6475         my $srcfd = new IO::File($confsrc, "r") ||
6476             die "unable to open file '$confsrc'\n";
6477
6478         my $outfd = new IO::File ($tmpfn, "w") ||
6479             die "unable to write config for VM $vmid\n";
6480
6481         my $cookie = { netcount => 0 };
6482         while (defined (my $line = <$srcfd>)) {
6483             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6484         }
6485
6486         $srcfd->close();
6487         $outfd->close();
6488     };
6489     if (my $err = $@) {
6490         unlink $tmpfn;
6491         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6492         die $err;
6493     }
6494
6495     rmtree $tmpdir;
6496
6497     rename $tmpfn, $conffile ||
6498         die "unable to commit configuration file '$conffile'\n";
6499
6500     PVE::Cluster::cfs_update(); # make sure we read new file
6501
6502     eval { rescan($vmid, 1); };
6503     warn $@ if $@;
6504 };
6505
6506 sub foreach_storage_used_by_vm {
6507     my ($conf, $func) = @_;
6508
6509     my $sidhash = {};
6510
6511     PVE::QemuConfig->foreach_volume($conf, sub {
6512         my ($ds, $drive) = @_;
6513         return if drive_is_cdrom($drive);
6514
6515         my $volid = $drive->{file};
6516
6517         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6518         $sidhash->{$sid} = $sid if $sid;
6519     });
6520
6521     foreach my $sid (sort keys %$sidhash) {
6522         &$func($sid);
6523     }
6524 }
6525
6526 my $qemu_snap_storage = {
6527     rbd => 1,
6528 };
6529 sub do_snapshots_with_qemu {
6530     my ($storecfg, $volid) = @_;
6531
6532     my $storage_name = PVE::Storage::parse_volume_id($volid);
6533     my $scfg = $storecfg->{ids}->{$storage_name};
6534
6535     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6536         return 1;
6537     }
6538
6539     if ($volid =~ m/\.(qcow2|qed)$/){
6540         return 1;
6541     }
6542
6543     return undef;
6544 }
6545
6546 sub qga_check_running {
6547     my ($vmid, $nowarn) = @_;
6548
6549     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6550     if ($@) {
6551         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6552         return 0;
6553     }
6554     return 1;
6555 }
6556
6557 sub template_create {
6558     my ($vmid, $conf, $disk) = @_;
6559
6560     my $storecfg = PVE::Storage::config();
6561
6562     PVE::QemuConfig->foreach_volume($conf, sub {
6563         my ($ds, $drive) = @_;
6564
6565         return if drive_is_cdrom($drive);
6566         return if $disk && $ds ne $disk;
6567
6568         my $volid = $drive->{file};
6569         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6570
6571         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6572         $drive->{file} = $voliddst;
6573         $conf->{$ds} = print_drive($drive);
6574         PVE::QemuConfig->write_config($vmid, $conf);
6575     });
6576 }
6577
6578 sub convert_iscsi_path {
6579     my ($path) = @_;
6580
6581     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6582         my $portal = $1;
6583         my $target = $2;
6584         my $lun = $3;
6585
6586         my $initiator_name = get_initiator_name();
6587
6588         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6589                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6590     }
6591
6592     die "cannot convert iscsi path '$path', unkown format\n";
6593 }
6594
6595 sub qemu_img_convert {
6596     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6597
6598     my $storecfg = PVE::Storage::config();
6599     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6600     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6601
6602     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6603
6604     my $cachemode;
6605     my $src_path;
6606     my $src_is_iscsi = 0;
6607     my $src_format;
6608
6609     if ($src_storeid) {
6610         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6611         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6612         $src_format = qemu_img_format($src_scfg, $src_volname);
6613         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6614         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6615         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6616     } elsif (-f $src_volid) {
6617         $src_path = $src_volid;
6618         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6619             $src_format = $1;
6620         }
6621     }
6622
6623     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6624
6625     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6626     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6627     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6628     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6629
6630     my $cmd = [];
6631     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6632     push @$cmd, '-l', "snapshot.name=$snapname"
6633         if $snapname && $src_format && $src_format eq "qcow2";
6634     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6635     push @$cmd, '-T', $cachemode if defined($cachemode);
6636
6637     if ($src_is_iscsi) {
6638         push @$cmd, '--image-opts';
6639         $src_path = convert_iscsi_path($src_path);
6640     } elsif ($src_format) {
6641         push @$cmd, '-f', $src_format;
6642     }
6643
6644     if ($dst_is_iscsi) {
6645         push @$cmd, '--target-image-opts';
6646         $dst_path = convert_iscsi_path($dst_path);
6647     } else {
6648         push @$cmd, '-O', $dst_format;
6649     }
6650
6651     push @$cmd, $src_path;
6652
6653     if (!$dst_is_iscsi && $is_zero_initialized) {
6654         push @$cmd, "zeroinit:$dst_path";
6655     } else {
6656         push @$cmd, $dst_path;
6657     }
6658
6659     my $parser = sub {
6660         my $line = shift;
6661         if($line =~ m/\((\S+)\/100\%\)/){
6662             my $percent = $1;
6663             my $transferred = int($size * $percent / 100);
6664             my $remaining = $size - $transferred;
6665
6666             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6667         }
6668
6669     };
6670
6671     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6672     my $err = $@;
6673     die "copy failed: $err" if $err;
6674 }
6675
6676 sub qemu_img_format {
6677     my ($scfg, $volname) = @_;
6678
6679     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6680         return $1;
6681     } else {
6682         return "raw";
6683     }
6684 }
6685
6686 sub qemu_drive_mirror {
6687     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6688
6689     $jobs = {} if !$jobs;
6690
6691     my $qemu_target;
6692     my $format;
6693     $jobs->{"drive-$drive"} = {};
6694
6695     if ($dst_volid =~ /^nbd:/) {
6696         $qemu_target = $dst_volid;
6697         $format = "nbd";
6698     } else {
6699         my $storecfg = PVE::Storage::config();
6700         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6701
6702         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6703
6704         $format = qemu_img_format($dst_scfg, $dst_volname);
6705
6706         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6707
6708         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6709     }
6710
6711     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6712     $opts->{format} = $format if $format;
6713
6714     if (defined($src_bitmap)) {
6715         $opts->{sync} = 'incremental';
6716         $opts->{bitmap} = $src_bitmap;
6717         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6718     }
6719
6720     if (defined($bwlimit)) {
6721         $opts->{speed} = $bwlimit * 1024;
6722         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6723     } else {
6724         print "drive mirror is starting for drive-$drive\n";
6725     }
6726
6727     # if a job already runs for this device we get an error, catch it for cleanup
6728     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6729     if (my $err = $@) {
6730         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6731         warn "$@\n" if $@;
6732         die "mirroring error: $err\n";
6733     }
6734
6735     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6736 }
6737
6738 # $completion can be either
6739 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6740 # 'cancel': wait until all jobs are ready, block-job-cancel them
6741 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6742 sub qemu_drive_mirror_monitor {
6743     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6744
6745     $completion //= 'complete';
6746
6747     eval {
6748         my $err_complete = 0;
6749
6750         while (1) {
6751             die "storage migration timed out\n" if $err_complete > 300;
6752
6753             my $stats = mon_cmd($vmid, "query-block-jobs");
6754
6755             my $running_mirror_jobs = {};
6756             foreach my $stat (@$stats) {
6757                 next if $stat->{type} ne 'mirror';
6758                 $running_mirror_jobs->{$stat->{device}} = $stat;
6759             }
6760
6761             my $readycounter = 0;
6762
6763             foreach my $job (keys %$jobs) {
6764
6765                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6766                     print "$job : finished\n";
6767                     delete $jobs->{$job};
6768                     next;
6769                 }
6770
6771                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6772
6773                 my $busy = $running_mirror_jobs->{$job}->{busy};
6774                 my $ready = $running_mirror_jobs->{$job}->{ready};
6775                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6776                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6777                     my $remaining = $total - $transferred;
6778                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6779
6780                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6781                 }
6782
6783                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6784             }
6785
6786             last if scalar(keys %$jobs) == 0;
6787
6788             if ($readycounter == scalar(keys %$jobs)) {
6789                 print "all mirroring jobs are ready \n";
6790                 last if $completion eq 'skip'; #do the complete later
6791
6792                 if ($vmiddst && $vmiddst != $vmid) {
6793                     my $agent_running = $qga && qga_check_running($vmid);
6794                     if ($agent_running) {
6795                         print "freeze filesystem\n";
6796                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6797                     } else {
6798                         print "suspend vm\n";
6799                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6800                     }
6801
6802                     # if we clone a disk for a new target vm, we don't switch the disk
6803                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6804
6805                     if ($agent_running) {
6806                         print "unfreeze filesystem\n";
6807                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6808                     } else {
6809                         print "resume vm\n";
6810                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6811                     }
6812
6813                     last;
6814                 } else {
6815
6816                     foreach my $job (keys %$jobs) {
6817                         # try to switch the disk if source and destination are on the same guest
6818                         print "$job: Completing block job...\n";
6819
6820                         my $op;
6821                         if ($completion eq 'complete') {
6822                             $op = 'block-job-complete';
6823                         } elsif ($completion eq 'cancel') {
6824                             $op = 'block-job-cancel';
6825                         } else {
6826                             die "invalid completion value: $completion\n";
6827                         }
6828                         eval { mon_cmd($vmid, $op, device => $job) };
6829                         if ($@ =~ m/cannot be completed/) {
6830                             print "$job: Block job cannot be completed, try again.\n";
6831                             $err_complete++;
6832                         }else {
6833                             print "$job: Completed successfully.\n";
6834                             $jobs->{$job}->{complete} = 1;
6835                         }
6836                     }
6837                 }
6838             }
6839             sleep 1;
6840         }
6841     };
6842     my $err = $@;
6843
6844     if ($err) {
6845         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6846         die "mirroring error: $err";
6847     }
6848
6849 }
6850
6851 sub qemu_blockjobs_cancel {
6852     my ($vmid, $jobs) = @_;
6853
6854     foreach my $job (keys %$jobs) {
6855         print "$job: Cancelling block job\n";
6856         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6857         $jobs->{$job}->{cancel} = 1;
6858     }
6859
6860     while (1) {
6861         my $stats = mon_cmd($vmid, "query-block-jobs");
6862
6863         my $running_jobs = {};
6864         foreach my $stat (@$stats) {
6865             $running_jobs->{$stat->{device}} = $stat;
6866         }
6867
6868         foreach my $job (keys %$jobs) {
6869
6870             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6871                 print "$job: Done.\n";
6872                 delete $jobs->{$job};
6873             }
6874         }
6875
6876         last if scalar(keys %$jobs) == 0;
6877
6878         sleep 1;
6879     }
6880 }
6881
6882 sub clone_disk {
6883     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6884         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6885
6886     my $newvolid;
6887
6888     if (!$full) {
6889         print "create linked clone of drive $drivename ($drive->{file})\n";
6890         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6891         push @$newvollist, $newvolid;
6892     } else {
6893
6894         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6895         $storeid = $storage if $storage;
6896
6897         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6898         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6899
6900         print "create full clone of drive $drivename ($drive->{file})\n";
6901         my $name = undef;
6902         if (drive_is_cloudinit($drive)) {
6903             $name = "vm-$newvmid-cloudinit";
6904             $name .= ".$dst_format" if $dst_format ne 'raw';
6905             $snapname = undef;
6906             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6907         } elsif ($drivename eq 'efidisk0') {
6908             $size = get_efivars_size($conf);
6909         }
6910         $size /= 1024;
6911         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
6912         push @$newvollist, $newvolid;
6913
6914         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6915
6916         if (drive_is_cloudinit($drive)) {
6917             goto no_data_clone;
6918         }
6919
6920         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6921         if (!$running || $snapname) {
6922             # TODO: handle bwlimits
6923             if ($drivename eq 'efidisk0') {
6924                 # the relevant data on the efidisk may be smaller than the source
6925                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6926                 # that is given by the OVMF_VARS.fd
6927                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6928                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6929                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6930                     "if=$src_path", "of=$dst_path"]);
6931             } else {
6932                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6933             }
6934         } else {
6935
6936             my $kvmver = get_running_qemu_version ($vmid);
6937             if (!min_version($kvmver, 2, 7)) {
6938                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6939                     if $drive->{iothread};
6940             }
6941
6942             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6943                 $completion, $qga, $bwlimit);
6944         }
6945     }
6946
6947 no_data_clone:
6948     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6949
6950     my $disk = $drive;
6951     $disk->{format} = undef;
6952     $disk->{file} = $newvolid;
6953     $disk->{size} = $size;
6954
6955     return $disk;
6956 }
6957
6958 sub get_running_qemu_version {
6959     my ($vmid) = @_;
6960     my $res = mon_cmd($vmid, "query-version");
6961     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6962 }
6963
6964 sub qemu_use_old_bios_files {
6965     my ($machine_type) = @_;
6966
6967     return if !$machine_type;
6968
6969     my $use_old_bios_files = undef;
6970
6971     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6972         $machine_type = $1;
6973         $use_old_bios_files = 1;
6974     } else {
6975         my $version = extract_version($machine_type, kvm_user_version());
6976         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6977         # load new efi bios files on migration. So this hack is required to allow
6978         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6979         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6980         $use_old_bios_files = !min_version($version, 2, 4);
6981     }
6982
6983     return ($use_old_bios_files, $machine_type);
6984 }
6985
6986 sub get_efivars_size {
6987     my ($conf) = @_;
6988     my $arch = get_vm_arch($conf);
6989     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6990     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
6991     return -s $ovmf_vars;
6992 }
6993
6994 sub update_efidisk_size {
6995     my ($conf) = @_;
6996
6997     return if !defined($conf->{efidisk0});
6998
6999     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7000     $disk->{size} = get_efivars_size($conf);
7001     $conf->{efidisk0} = print_drive($disk);
7002
7003     return;
7004 }
7005
7006 sub create_efidisk($$$$$) {
7007     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7008
7009     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7010     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7011
7012     my $vars_size_b = -s $ovmf_vars;
7013     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7014     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7015     PVE::Storage::activate_volumes($storecfg, [$volid]);
7016
7017     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7018     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7019
7020     return ($volid, $size/1024);
7021 }
7022
7023 sub vm_iothreads_list {
7024     my ($vmid) = @_;
7025
7026     my $res = mon_cmd($vmid, 'query-iothreads');
7027
7028     my $iothreads = {};
7029     foreach my $iothread (@$res) {
7030         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7031     }
7032
7033     return $iothreads;
7034 }
7035
7036 sub scsihw_infos {
7037     my ($conf, $drive) = @_;
7038
7039     my $maxdev = 0;
7040
7041     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7042         $maxdev = 7;
7043     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7044         $maxdev = 1;
7045     } else {
7046         $maxdev = 256;
7047     }
7048
7049     my $controller = int($drive->{index} / $maxdev);
7050     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7051         ? "virtioscsi"
7052         : "scsihw";
7053
7054     return ($maxdev, $controller, $controller_prefix);
7055 }
7056
7057 sub windows_version {
7058     my ($ostype) = @_;
7059
7060     return 0 if !$ostype;
7061
7062     my $winversion = 0;
7063
7064     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7065         $winversion = 5;
7066     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7067         $winversion = 6;
7068     } elsif ($ostype =~ m/^win(\d+)$/) {
7069         $winversion = $1;
7070     }
7071
7072     return $winversion;
7073 }
7074
7075 sub resolve_dst_disk_format {
7076         my ($storecfg, $storeid, $src_volname, $format) = @_;
7077         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7078
7079         if (!$format) {
7080             # if no target format is specified, use the source disk format as hint
7081             if ($src_volname) {
7082                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7083                 $format = qemu_img_format($scfg, $src_volname);
7084             } else {
7085                 return $defFormat;
7086             }
7087         }
7088
7089         # test if requested format is supported - else use default
7090         my $supported = grep { $_ eq $format } @$validFormats;
7091         $format = $defFormat if !$supported;
7092         return $format;
7093 }
7094
7095 # NOTE: if this logic changes, please update docs & possibly gui logic
7096 sub find_vmstate_storage {
7097     my ($conf, $storecfg) = @_;
7098
7099     # first, return storage from conf if set
7100     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7101
7102     my ($target, $shared, $local);
7103
7104     foreach_storage_used_by_vm($conf, sub {
7105         my ($sid) = @_;
7106         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7107         my $dst = $scfg->{shared} ? \$shared : \$local;
7108         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7109     });
7110
7111     # second, use shared storage where VM has at least one disk
7112     # third, use local storage where VM has at least one disk
7113     # fall back to local storage
7114     $target = $shared // $local // 'local';
7115
7116     return $target;
7117 }
7118
7119 sub generate_uuid {
7120     my ($uuid, $uuid_str);
7121     UUID::generate($uuid);
7122     UUID::unparse($uuid, $uuid_str);
7123     return $uuid_str;
7124 }
7125
7126 sub generate_smbios1_uuid {
7127     return "uuid=".generate_uuid();
7128 }
7129
7130 sub nbd_stop {
7131     my ($vmid) = @_;
7132
7133     mon_cmd($vmid, 'nbd-server-stop');
7134 }
7135
7136 sub create_reboot_request {
7137     my ($vmid) = @_;
7138     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7139         or die "failed to create reboot trigger file: $!\n";
7140     close($fh);
7141 }
7142
7143 sub clear_reboot_request {
7144     my ($vmid) = @_;
7145     my $path = "/run/qemu-server/$vmid.reboot";
7146     my $res = 0;
7147
7148     $res = unlink($path);
7149     die "could not remove reboot request for $vmid: $!"
7150         if !$res && $! != POSIX::ENOENT;
7151
7152     return $res;
7153 }
7154
7155 # bash completion helper
7156
7157 sub complete_backup_archives {
7158     my ($cmdname, $pname, $cvalue) = @_;
7159
7160     my $cfg = PVE::Storage::config();
7161
7162     my $storeid;
7163
7164     if ($cvalue =~ m/^([^:]+):/) {
7165         $storeid = $1;
7166     }
7167
7168     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7169
7170     my $res = [];
7171     foreach my $id (keys %$data) {
7172         foreach my $item (@{$data->{$id}}) {
7173             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7174             push @$res, $item->{volid} if defined($item->{volid});
7175         }
7176     }
7177
7178     return $res;
7179 }
7180
7181 my $complete_vmid_full = sub {
7182     my ($running) = @_;
7183
7184     my $idlist = vmstatus();
7185
7186     my $res = [];
7187
7188     foreach my $id (keys %$idlist) {
7189         my $d = $idlist->{$id};
7190         if (defined($running)) {
7191             next if $d->{template};
7192             next if $running && $d->{status} ne 'running';
7193             next if !$running && $d->{status} eq 'running';
7194         }
7195         push @$res, $id;
7196
7197     }
7198     return $res;
7199 };
7200
7201 sub complete_vmid {
7202     return &$complete_vmid_full();
7203 }
7204
7205 sub complete_vmid_stopped {
7206     return &$complete_vmid_full(0);
7207 }
7208
7209 sub complete_vmid_running {
7210     return &$complete_vmid_full(1);
7211 }
7212
7213 sub complete_storage {
7214
7215     my $cfg = PVE::Storage::config();
7216     my $ids = $cfg->{ids};
7217
7218     my $res = [];
7219     foreach my $sid (keys %$ids) {
7220         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7221         next if !$ids->{$sid}->{content}->{images};
7222         push @$res, $sid;
7223     }
7224
7225     return $res;
7226 }
7227
7228 sub complete_migration_storage {
7229     my ($cmd, $param, $current_value, $all_args) = @_;
7230
7231     my $targetnode = @$all_args[1];
7232
7233     my $cfg = PVE::Storage::config();
7234     my $ids = $cfg->{ids};
7235
7236     my $res = [];
7237     foreach my $sid (keys %$ids) {
7238         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7239         next if !$ids->{$sid}->{content}->{images};
7240         push @$res, $sid;
7241     }
7242
7243     return $res;
7244 }
7245
7246 1;