PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Machine;
  48 use PVE::QemuServer::Memory;
  49 use PVE::QemuServer::Monitor qw(mon_cmd);
  50 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  51 use PVE::QemuServer::USB qw(parse_usb_device);
  52
  53 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  54 my $OVMF = {
  55     x86_64 => [
  56         "$EDK2_FW_BASE/OVMF_CODE.fd",
  57         "$EDK2_FW_BASE/OVMF_VARS.fd"
  58     ],
  59     aarch64 => [
  60         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  61         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  62     ],
  63 };
  64
  65 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  66
  67 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  68
  69 # Note about locking: we use flock on the config file protect
  70 # against concurent actions.
  71 # Aditionaly, we have a 'lock' setting in the config file. This
  72 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  73 # allowed when such lock is set. But you can ignore this kind of
  74 # lock with the --skiplock flag.
  75
  76 cfs_register_file('/qemu-server/',
  77                   \&parse_vm_config,
  78                   \&write_vm_config);
  79
  80 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  81     description => "Some command save/restore state from this location.",
  82     type => 'string',
  83     maxLength => 128,
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  88     type => 'string',
  89     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  90     description => "The drive's backing file's data format.",
  91     optional => 1,
  92 });
  93
  94 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  95         description => "Specifies the Qemu machine type.",
  96         type => 'string',
  97         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  98         maxLength => 40,
  99         optional => 1,
 100 });
 101
 102 #no warnings 'redefine';
 103
 104 sub cgroups_write {
 105    my ($controller, $vmid, $option, $value) = @_;
 106
 107    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 108    PVE::ProcFSTools::write_proc_entry($path, $value);
 109
 110 }
 111
 112 my $nodename_cache;
 113 sub nodename {
 114     $nodename_cache //= PVE::INotify::nodename();
 115     return $nodename_cache;
 116 }
 117
 118 my $watchdog_fmt = {
 119     model => {
 120         default_key => 1,
 121         type => 'string',
 122         enum => [qw(i6300esb ib700)],
 123         description => "Watchdog type to emulate.",
 124         default => 'i6300esb',
 125         optional => 1,
 126     },
 127     action => {
 128         type => 'string',
 129         enum => [qw(reset shutdown poweroff pause debug none)],
 130         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 131         optional => 1,
 132     },
 133 };
 134 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 135
 136 my $agent_fmt = {
 137     enabled => {
 138         description => "Enable/disable Qemu GuestAgent.",
 139         type => 'boolean',
 140         default => 0,
 141         default_key => 1,
 142     },
 143     fstrim_cloned_disks => {
 144         description => "Run fstrim after cloning/moving a disk.",
 145         type => 'boolean',
 146         optional => 1,
 147         default => 0
 148     },
 149     type => {
 150         description => "Select the agent type",
 151         type => 'string',
 152         default => 'virtio',
 153         optional => 1,
 154         enum => [qw(virtio isa)],
 155     },
 156 };
 157
 158 my $vga_fmt = {
 159     type => {
 160         description => "Select the VGA type.",
 161         type => 'string',
 162         default => 'std',
 163         optional => 1,
 164         default_key => 1,
 165         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 166     },
 167     memory => {
 168         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 169         type => 'integer',
 170         optional => 1,
 171         minimum => 4,
 172         maximum => 512,
 173     },
 174 };
 175
 176 my $ivshmem_fmt = {
 177     size => {
 178         type => 'integer',
 179         minimum => 1,
 180         description => "The size of the file in MB.",
 181     },
 182     name => {
 183         type => 'string',
 184         pattern => '[a-zA-Z0-9\-]+',
 185         optional => 1,
 186         format_description => 'string',
 187         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 188     },
 189 };
 190
 191 my $audio_fmt = {
 192     device => {
 193         type => 'string',
 194         enum => [qw(ich9-intel-hda intel-hda AC97)],
 195         description =>  "Configure an audio device."
 196     },
 197     driver =>  {
 198         type => 'string',
 199         enum => ['spice'],
 200         default => 'spice',
 201         optional => 1,
 202         description => "Driver backend for the audio device."
 203     },
 204 };
 205
 206 my $spice_enhancements_fmt = {
 207     foldersharing => {
 208         type => 'boolean',
 209         optional => 1,
 210         default => '0',
 211         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 212     },
 213     videostreaming =>  {
 214         type => 'string',
 215         enum => ['off', 'all', 'filter'],
 216         default => 'off',
 217         optional => 1,
 218         description => "Enable video streaming. Uses compression for detected video streams."
 219     },
 220 };
 221
 222 my $confdesc = {
 223     onboot => {
 224         optional => 1,
 225         type => 'boolean',
 226         description => "Specifies whether a VM will be started during system bootup.",
 227         default => 0,
 228     },
 229     autostart => {
 230         optional => 1,
 231         type => 'boolean',
 232         description => "Automatic restart after crash (currently ignored).",
 233         default => 0,
 234     },
 235     hotplug => {
 236         optional => 1,
 237         type => 'string', format => 'pve-hotplug-features',
 238         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 239         default => 'network,disk,usb',
 240     },
 241     reboot => {
 242         optional => 1,
 243         type => 'boolean',
 244         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 245         default => 1,
 246     },
 247     lock => {
 248         optional => 1,
 249         type => 'string',
 250         description => "Lock/unlock the VM.",
 251         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 252     },
 253     cpulimit => {
 254         optional => 1,
 255         type => 'number',
 256         description => "Limit of CPU usage.",
 257         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 258         minimum => 0,
 259         maximum => 128,
 260         default => 0,
 261     },
 262     cpuunits => {
 263         optional => 1,
 264         type => 'integer',
 265         description => "CPU weight for a VM.",
 266         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 267         minimum => 2,
 268         maximum => 262144,
 269         default => 1024,
 270     },
 271     memory => {
 272         optional => 1,
 273         type => 'integer',
 274         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 275         minimum => 16,
 276         default => 512,
 277     },
 278     balloon => {
 279         optional => 1,
 280         type => 'integer',
 281         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 282         minimum => 0,
 283     },
 284     shares => {
 285         optional => 1,
 286         type => 'integer',
 287         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 288         minimum => 0,
 289         maximum => 50000,
 290         default => 1000,
 291     },
 292     keyboard => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 296                        "It should not be necessary to set it.",
 297         enum => PVE::Tools::kvmkeymaplist(),
 298         default => undef,
 299     },
 300     name => {
 301         optional => 1,
 302         type => 'string', format => 'dns-name',
 303         description => "Set a name for the VM. Only used on the configuration web interface.",
 304     },
 305     scsihw => {
 306         optional => 1,
 307         type => 'string',
 308         description => "SCSI controller model",
 309         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 310         default => 'lsi',
 311     },
 312     description => {
 313         optional => 1,
 314         type => 'string',
 315         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 316     },
 317     ostype => {
 318         optional => 1,
 319         type => 'string',
 320         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 321         description => "Specify guest operating system.",
 322         verbose_description => <<EODESC,
 323 Specify guest operating system. This is used to enable special
 324 optimization/features for specific operating systems:
 325
 326 [horizontal]
 327 other;; unspecified OS
 328 wxp;; Microsoft Windows XP
 329 w2k;; Microsoft Windows 2000
 330 w2k3;; Microsoft Windows 2003
 331 w2k8;; Microsoft Windows 2008
 332 wvista;; Microsoft Windows Vista
 333 win7;; Microsoft Windows 7
 334 win8;; Microsoft Windows 8/2012/2012r2
 335 win10;; Microsoft Windows 10/2016
 336 l24;; Linux 2.4 Kernel
 337 l26;; Linux 2.6 - 5.X Kernel
 338 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 339 EODESC
 340     },
 341     boot => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 345         pattern => '[acdn]{1,4}',
 346         default => 'cdn',
 347     },
 348     bootdisk => {
 349         optional => 1,
 350         type => 'string', format => 'pve-qm-bootdisk',
 351         description => "Enable booting from specified disk.",
 352         pattern => '(ide|sata|scsi|virtio)\d+',
 353     },
 354     smp => {
 355         optional => 1,
 356         type => 'integer',
 357         description => "The number of CPUs. Please use option -sockets instead.",
 358         minimum => 1,
 359         default => 1,
 360     },
 361     sockets => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "The number of CPU sockets.",
 365         minimum => 1,
 366         default => 1,
 367     },
 368     cores => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "The number of cores per socket.",
 372         minimum => 1,
 373         default => 1,
 374     },
 375     numa => {
 376         optional => 1,
 377         type => 'boolean',
 378         description => "Enable/disable NUMA.",
 379         default => 0,
 380     },
 381     hugepages => {
 382         optional => 1,
 383         type => 'string',
 384         description => "Enable/disable hugepages memory.",
 385         enum => [qw(any 2 1024)],
 386     },
 387     vcpus => {
 388         optional => 1,
 389         type => 'integer',
 390         description => "Number of hotplugged vcpus.",
 391         minimum => 1,
 392         default => 0,
 393     },
 394     acpi => {
 395         optional => 1,
 396         type => 'boolean',
 397         description => "Enable/disable ACPI.",
 398         default => 1,
 399     },
 400     agent => {
 401         optional => 1,
 402         description => "Enable/disable Qemu GuestAgent and its properties.",
 403         type => 'string',
 404         format => $agent_fmt,
 405     },
 406     kvm => {
 407         optional => 1,
 408         type => 'boolean',
 409         description => "Enable/disable KVM hardware virtualization.",
 410         default => 1,
 411     },
 412     tdf => {
 413         optional => 1,
 414         type => 'boolean',
 415         description => "Enable/disable time drift fix.",
 416         default => 0,
 417     },
 418     localtime => {
 419         optional => 1,
 420         type => 'boolean',
 421         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 422     },
 423     freeze => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 427     },
 428     vga => {
 429         optional => 1,
 430         type => 'string', format => $vga_fmt,
 431         description => "Configure the VGA hardware.",
 432         verbose_description => "Configure the VGA Hardware. If you want to use ".
 433             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 434             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 435             "is 'std' for all OS types besides some Windows versions (XP and " .
 436             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 437             "display server. For win* OS you can select how many independent " .
 438             "displays you want, Linux guests can add displays them self.\n".
 439             "You can also run without any graphic card, using a serial device as terminal.",
 440     },
 441     watchdog => {
 442         optional => 1,
 443         type => 'string', format => 'pve-qm-watchdog',
 444         description => "Create a virtual hardware watchdog device.",
 445         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 446             " (by a guest action), the watchdog must be periodically polled " .
 447             "by an agent inside the guest or else the watchdog will reset " .
 448             "the guest (or execute the respective action specified)",
 449     },
 450     startdate => {
 451         optional => 1,
 452         type => 'string',
 453         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 454         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 455         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 456         default => 'now',
 457     },
 458     startup =>  get_standard_option('pve-startup-order'),
 459     template => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Enable/disable Template.",
 463         default => 0,
 464     },
 465     args => {
 466         optional => 1,
 467         type => 'string',
 468         description => "Arbitrary arguments passed to kvm.",
 469         verbose_description => <<EODESCR,
 470 Arbitrary arguments passed to kvm, for example:
 471
 472 args: -no-reboot -no-hpet
 473
 474 NOTE: this option is for experts only.
 475 EODESCR
 476     },
 477     tablet => {
 478         optional => 1,
 479         type => 'boolean',
 480         default => 1,
 481         description => "Enable/disable the USB tablet device.",
 482         verbose_description => "Enable/disable the USB tablet device. This device is " .
 483             "usually needed to allow absolute mouse positioning with VNC. " .
 484             "Else the mouse runs out of sync with normal VNC clients. " .
 485             "If you're running lots of console-only guests on one host, " .
 486             "you may consider disabling this to save some context switches. " .
 487             "This is turned off by default if you use spice (-vga=qxl).",
 488     },
 489     migrate_speed => {
 490         optional => 1,
 491         type => 'integer',
 492         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 493         minimum => 0,
 494         default => 0,
 495     },
 496     migrate_downtime => {
 497         optional => 1,
 498         type => 'number',
 499         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 500         minimum => 0,
 501         default => 0.1,
 502     },
 503     cdrom => {
 504         optional => 1,
 505         type => 'string', format => 'pve-qm-ide',
 506         typetext => '<volume>',
 507         description => "This is an alias for option -ide2",
 508     },
 509     cpu => {
 510         optional => 1,
 511         description => "Emulated CPU type.",
 512         type => 'string',
 513         format => $PVE::QemuServer::CPUConfig::cpu_fmt,
 514     },
 515     parent => get_standard_option('pve-snapshot-name', {
 516         optional => 1,
 517         description => "Parent snapshot name. This is used internally, and should not be modified.",
 518     }),
 519     snaptime => {
 520         optional => 1,
 521         description => "Timestamp for snapshots.",
 522         type => 'integer',
 523         minimum => 0,
 524     },
 525     vmstate => {
 526         optional => 1,
 527         type => 'string', format => 'pve-volume-id',
 528         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 529     },
 530     vmstatestorage => get_standard_option('pve-storage-id', {
 531         description => "Default storage for VM state volumes/files.",
 532         optional => 1,
 533     }),
 534     runningmachine => get_standard_option('pve-qemu-machine', {
 535         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 536     }),
 537     machine => get_standard_option('pve-qemu-machine'),
 538     arch => {
 539         description => "Virtual processor architecture. Defaults to the host.",
 540         optional => 1,
 541         type => 'string',
 542         enum => [qw(x86_64 aarch64)],
 543     },
 544     smbios1 => {
 545         description => "Specify SMBIOS type 1 fields.",
 546         type => 'string', format => 'pve-qm-smbios1',
 547         maxLength => 512,
 548         optional => 1,
 549     },
 550     protection => {
 551         optional => 1,
 552         type => 'boolean',
 553         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 554         default => 0,
 555     },
 556     bios => {
 557         optional => 1,
 558         type => 'string',
 559         enum => [ qw(seabios ovmf) ],
 560         description => "Select BIOS implementation.",
 561         default => 'seabios',
 562     },
 563     vmgenid => {
 564         type => 'string',
 565         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 566         format_description => 'UUID',
 567         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 568         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 569             " 128-bit integer value identifier to the guest OS. This allows to".
 570             " notify the guest operating system when the virtual machine is".
 571             " executed with a different configuration (e.g. snapshot execution".
 572             " or creation from a template). The guest operating system notices".
 573             " the change, and is then able to react as appropriate by marking".
 574             " its copies of distributed databases as dirty, re-initializing its".
 575             " random number generator, etc.\n".
 576             "Note that auto-creation only works when done throug API/CLI create".
 577             " or update methods, but not when manually editing the config file.",
 578         default => "1 (autogenerated)",
 579         optional => 1,
 580     },
 581     hookscript => {
 582         type => 'string',
 583         format => 'pve-volume-id',
 584         optional => 1,
 585         description => "Script that will be executed during various steps in the vms lifetime.",
 586     },
 587     ivshmem => {
 588         type => 'string',
 589         format => $ivshmem_fmt,
 590         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 591         optional => 1,
 592     },
 593     audio0 => {
 594         type => 'string',
 595         format => $audio_fmt,
 596         description => "Configure a audio device, useful in combination with QXL/Spice.",
 597         optional => 1
 598     },
 599     spice_enhancements => {
 600         type => 'string',
 601         format => $spice_enhancements_fmt,
 602         description => "Configure additional enhancements for SPICE.",
 603         optional => 1
 604     },
 605     tags => {
 606         type => 'string', format => 'pve-tag-list',
 607         description => 'Tags of the VM. This is only meta information.',
 608         optional => 1,
 609     },
 610 };
 611
 612 my $cicustom_fmt = {
 613     meta => {
 614         type => 'string',
 615         optional => 1,
 616         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 617         format => 'pve-volume-id',
 618         format_description => 'volume',
 619     },
 620     network => {
 621         type => 'string',
 622         optional => 1,
 623         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 624         format => 'pve-volume-id',
 625         format_description => 'volume',
 626     },
 627     user => {
 628         type => 'string',
 629         optional => 1,
 630         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 631         format => 'pve-volume-id',
 632         format_description => 'volume',
 633     },
 634 };
 635 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 636
 637 my $confdesc_cloudinit = {
 638     citype => {
 639         optional => 1,
 640         type => 'string',
 641         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 642         enum => ['configdrive2', 'nocloud'],
 643     },
 644     ciuser => {
 645         optional => 1,
 646         type => 'string',
 647         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 648     },
 649     cipassword => {
 650         optional => 1,
 651         type => 'string',
 652         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 653     },
 654     cicustom => {
 655         optional => 1,
 656         type => 'string',
 657         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 658         format => 'pve-qm-cicustom',
 659     },
 660     searchdomain => {
 661         optional => 1,
 662         type => 'string',
 663         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 664     },
 665     nameserver => {
 666         optional => 1,
 667         type => 'string', format => 'address-list',
 668         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 669     },
 670     sshkeys => {
 671         optional => 1,
 672         type => 'string',
 673         format => 'urlencoded',
 674         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 675     },
 676 };
 677
 678 # what about other qemu settings ?
 679 #cpu => 'string',
 680 #machine => 'string',
 681 #fda => 'file',
 682 #fdb => 'file',
 683 #mtdblock => 'file',
 684 #sd => 'file',
 685 #pflash => 'file',
 686 #snapshot => 'bool',
 687 #bootp => 'file',
 688 ##tftp => 'dir',
 689 ##smb => 'dir',
 690 #kernel => 'file',
 691 #append => 'string',
 692 #initrd => 'file',
 693 ##soundhw => 'string',
 694
 695 while (my ($k, $v) = each %$confdesc) {
 696     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 697 }
 698
 699 my $MAX_IDE_DISKS = 4;
 700 my $MAX_SCSI_DISKS = 31;
 701 my $MAX_VIRTIO_DISKS = 16;
 702 my $MAX_SATA_DISKS = 6;
 703 my $MAX_USB_DEVICES = 5;
 704 my $MAX_NETS = 32;
 705 my $MAX_UNUSED_DISKS = 256;
 706 my $MAX_HOSTPCI_DEVICES = 16;
 707 my $MAX_SERIAL_PORTS = 4;
 708 my $MAX_PARALLEL_PORTS = 3;
 709 my $MAX_NUMA = 8;
 710
 711 my $numa_fmt = {
 712     cpus => {
 713         type => "string",
 714         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 715         description => "CPUs accessing this NUMA node.",
 716         format_description => "id[-id];...",
 717     },
 718     memory => {
 719         type => "number",
 720         description => "Amount of memory this NUMA node provides.",
 721         optional => 1,
 722     },
 723     hostnodes => {
 724         type => "string",
 725         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 726         description => "Host NUMA nodes to use.",
 727         format_description => "id[-id];...",
 728         optional => 1,
 729     },
 730     policy => {
 731         type => 'string',
 732         enum => [qw(preferred bind interleave)],
 733         description => "NUMA allocation policy.",
 734         optional => 1,
 735     },
 736 };
 737 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 738 my $numadesc = {
 739     optional => 1,
 740     type => 'string', format => $numa_fmt,
 741     description => "NUMA topology.",
 742 };
 743 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 744
 745 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 746     $confdesc->{"numa$i"} = $numadesc;
 747 }
 748
 749 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 750                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 751                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 752 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 753
 754 my $net_fmt_bridge_descr = <<__EOD__;
 755 Bridge to attach the network device to. The Proxmox VE standard bridge
 756 is called 'vmbr0'.
 757
 758 If you do not specify a bridge, we create a kvm user (NATed) network
 759 device, which provides DHCP and DNS services. The following addresses
 760 are used:
 761
 762  10.0.2.2   Gateway
 763  10.0.2.3   DNS Server
 764  10.0.2.4   SMB Server
 765
 766 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 767 __EOD__
 768
 769 my $net_fmt = {
 770     macaddr  => get_standard_option('mac-addr', {
 771         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 772     }),
 773     model => {
 774         type => 'string',
 775         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 776         enum => $nic_model_list,
 777         default_key => 1,
 778     },
 779     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 780     bridge => {
 781         type => 'string',
 782         description => $net_fmt_bridge_descr,
 783         format_description => 'bridge',
 784         optional => 1,
 785     },
 786     queues => {
 787         type => 'integer',
 788         minimum => 0, maximum => 16,
 789         description => 'Number of packet queues to be used on the device.',
 790         optional => 1,
 791     },
 792     rate => {
 793         type => 'number',
 794         minimum => 0,
 795         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 796         optional => 1,
 797     },
 798     tag => {
 799         type => 'integer',
 800         minimum => 1, maximum => 4094,
 801         description => 'VLAN tag to apply to packets on this interface.',
 802         optional => 1,
 803     },
 804     trunks => {
 805         type => 'string',
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => 'VLAN trunks to pass through this interface.',
 808         format_description => 'vlanid[;vlanid...]',
 809         optional => 1,
 810     },
 811     firewall => {
 812         type => 'boolean',
 813         description => 'Whether this interface should be protected by the firewall.',
 814         optional => 1,
 815     },
 816     link_down => {
 817         type => 'boolean',
 818         description => 'Whether this interface should be disconnected (like pulling the plug).',
 819         optional => 1,
 820     },
 821 };
 822
 823 my $netdesc = {
 824     optional => 1,
 825     type => 'string', format => $net_fmt,
 826     description => "Specify network devices.",
 827 };
 828
 829 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 830
 831 my $ipconfig_fmt = {
 832     ip => {
 833         type => 'string',
 834         format => 'pve-ipv4-config',
 835         format_description => 'IPv4Format/CIDR',
 836         description => 'IPv4 address in CIDR format.',
 837         optional => 1,
 838         default => 'dhcp',
 839     },
 840     gw => {
 841         type => 'string',
 842         format => 'ipv4',
 843         format_description => 'GatewayIPv4',
 844         description => 'Default gateway for IPv4 traffic.',
 845         optional => 1,
 846         requires => 'ip',
 847     },
 848     ip6 => {
 849         type => 'string',
 850         format => 'pve-ipv6-config',
 851         format_description => 'IPv6Format/CIDR',
 852         description => 'IPv6 address in CIDR format.',
 853         optional => 1,
 854         default => 'dhcp',
 855     },
 856     gw6 => {
 857         type => 'string',
 858         format => 'ipv6',
 859         format_description => 'GatewayIPv6',
 860         description => 'Default gateway for IPv6 traffic.',
 861         optional => 1,
 862         requires => 'ip6',
 863     },
 864 };
 865 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 866 my $ipconfigdesc = {
 867     optional => 1,
 868     type => 'string', format => 'pve-qm-ipconfig',
 869     description => <<'EODESCR',
 870 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 871
 872 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 873
 874 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 875 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 876
 877 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 878 EODESCR
 879 };
 880 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 881
 882 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 883     $confdesc->{"net$i"} = $netdesc;
 884     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 885 }
 886
 887 foreach my $key (keys %$confdesc_cloudinit) {
 888     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 889 }
 890
 891 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 892 sub verify_volume_id_or_qm_path {
 893     my ($volid, $noerr) = @_;
 894
 895     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 896         return $volid;
 897     }
 898
 899     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 900     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 901     if ($@) {
 902         return undef if $noerr;
 903         die $@;
 904     }
 905     return $volid;
 906 }
 907
 908 my $drivename_hash;
 909
 910 my %drivedesc_base = (
 911     volume => { alias => 'file' },
 912     file => {
 913         type => 'string',
 914         format => 'pve-volume-id-or-qm-path',
 915         default_key => 1,
 916         format_description => 'volume',
 917         description => "The drive's backing volume.",
 918     },
 919     media => {
 920         type => 'string',
 921         enum => [qw(cdrom disk)],
 922         description => "The drive's media type.",
 923         default => 'disk',
 924         optional => 1
 925     },
 926     cyls => {
 927         type => 'integer',
 928         description => "Force the drive's physical geometry to have a specific cylinder count.",
 929         optional => 1
 930     },
 931     heads => {
 932         type => 'integer',
 933         description => "Force the drive's physical geometry to have a specific head count.",
 934         optional => 1
 935     },
 936     secs => {
 937         type => 'integer',
 938         description => "Force the drive's physical geometry to have a specific sector count.",
 939         optional => 1
 940     },
 941     trans => {
 942         type => 'string',
 943         enum => [qw(none lba auto)],
 944         description => "Force disk geometry bios translation mode.",
 945         optional => 1,
 946     },
 947     snapshot => {
 948         type => 'boolean',
 949         description => "Controls qemu's snapshot mode feature."
 950             . " If activated, changes made to the disk are temporary and will"
 951             . " be discarded when the VM is shutdown.",
 952         optional => 1,
 953     },
 954     cache => {
 955         type => 'string',
 956         enum => [qw(none writethrough writeback unsafe directsync)],
 957         description => "The drive's cache mode",
 958         optional => 1,
 959     },
 960     format => get_standard_option('pve-qm-image-format'),
 961     size => {
 962         type => 'string',
 963         format => 'disk-size',
 964         format_description => 'DiskSize',
 965         description => "Disk size. This is purely informational and has no effect.",
 966         optional => 1,
 967     },
 968     backup => {
 969         type => 'boolean',
 970         description => "Whether the drive should be included when making backups.",
 971         optional => 1,
 972     },
 973     replicate => {
 974         type => 'boolean',
 975         description => 'Whether the drive should considered for replication jobs.',
 976         optional => 1,
 977         default => 1,
 978     },
 979     rerror => {
 980         type => 'string',
 981         enum => [qw(ignore report stop)],
 982         description => 'Read error action.',
 983         optional => 1,
 984     },
 985     werror => {
 986         type => 'string',
 987         enum => [qw(enospc ignore report stop)],
 988         description => 'Write error action.',
 989         optional => 1,
 990     },
 991     aio => {
 992         type => 'string',
 993         enum => [qw(native threads)],
 994         description => 'AIO type to use.',
 995         optional => 1,
 996     },
 997     discard => {
 998         type => 'string',
 999         enum => [qw(ignore on)],
1000         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1001         optional => 1,
1002     },
1003     detect_zeroes => {
1004         type => 'boolean',
1005         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1006         optional => 1,
1007     },
1008     serial => {
1009         type => 'string',
1010         format => 'urlencoded',
1011         format_description => 'serial',
1012         maxLength => 20*3, # *3 since it's %xx url enoded
1013         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1014         optional => 1,
1015     },
1016     shared => {
1017         type => 'boolean',
1018         description => 'Mark this locally-managed volume as available on all nodes',
1019         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1020         optional => 1,
1021         default => 0,
1022     }
1023 );
1024
1025 my %iothread_fmt = ( iothread => {
1026         type => 'boolean',
1027         description => "Whether to use iothreads for this drive",
1028         optional => 1,
1029 });
1030
1031 my %model_fmt = (
1032     model => {
1033         type => 'string',
1034         format => 'urlencoded',
1035         format_description => 'model',
1036         maxLength => 40*3, # *3 since it's %xx url enoded
1037         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1038         optional => 1,
1039     },
1040 );
1041
1042 my %queues_fmt = (
1043     queues => {
1044         type => 'integer',
1045         description => "Number of queues.",
1046         minimum => 2,
1047         optional => 1
1048     }
1049 );
1050
1051 my %scsiblock_fmt = (
1052     scsiblock => {
1053         type => 'boolean',
1054         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1055         optional => 1,
1056         default => 0,
1057     },
1058 );
1059
1060 my %ssd_fmt = (
1061     ssd => {
1062         type => 'boolean',
1063         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1064         optional => 1,
1065     },
1066 );
1067
1068 my %wwn_fmt = (
1069     wwn => {
1070         type => 'string',
1071         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1072         format_description => 'wwn',
1073         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1074         optional => 1,
1075     },
1076 );
1077
1078 my $add_throttle_desc = sub {
1079     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1080     my $d = {
1081         type => $type,
1082         format_description => $unit,
1083         description => "Maximum $what in $longunit.",
1084         optional => 1,
1085     };
1086     $d->{minimum} = $minimum if defined($minimum);
1087     $drivedesc_base{$key} = $d;
1088 };
1089 # throughput: (leaky bucket)
1090 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1091 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1092 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1093 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1094 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1095 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1096 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1097 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1098 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1099
1100 # pools: (pool of IO before throttling starts taking effect)
1101 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1102 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1103 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1104 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1105 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1106 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1107
1108 # burst lengths
1109 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1110 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1111 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1112 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1113 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1114 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1115
1116 # legacy support
1117 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1118 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1119 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1120 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1121
1122 my $ide_fmt = {
1123     %drivedesc_base,
1124     %model_fmt,
1125     %ssd_fmt,
1126     %wwn_fmt,
1127 };
1128 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1129
1130 my $idedesc = {
1131     optional => 1,
1132     type => 'string', format => $ide_fmt,
1133     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1134 };
1135 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1136
1137 my $scsi_fmt = {
1138     %drivedesc_base,
1139     %iothread_fmt,
1140     %queues_fmt,
1141     %scsiblock_fmt,
1142     %ssd_fmt,
1143     %wwn_fmt,
1144 };
1145 my $scsidesc = {
1146     optional => 1,
1147     type => 'string', format => $scsi_fmt,
1148     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1149 };
1150 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1151
1152 my $sata_fmt = {
1153     %drivedesc_base,
1154     %ssd_fmt,
1155     %wwn_fmt,
1156 };
1157 my $satadesc = {
1158     optional => 1,
1159     type => 'string', format => $sata_fmt,
1160     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1161 };
1162 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1163
1164 my $virtio_fmt = {
1165     %drivedesc_base,
1166     %iothread_fmt,
1167 };
1168 my $virtiodesc = {
1169     optional => 1,
1170     type => 'string', format => $virtio_fmt,
1171     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1172 };
1173 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1174
1175 my $alldrive_fmt = {
1176     %drivedesc_base,
1177     %iothread_fmt,
1178     %model_fmt,
1179     %queues_fmt,
1180     %scsiblock_fmt,
1181     %ssd_fmt,
1182     %wwn_fmt,
1183 };
1184
1185 my $efidisk_fmt = {
1186     volume => { alias => 'file' },
1187     file => {
1188         type => 'string',
1189         format => 'pve-volume-id-or-qm-path',
1190         default_key => 1,
1191         format_description => 'volume',
1192         description => "The drive's backing volume.",
1193     },
1194     format => get_standard_option('pve-qm-image-format'),
1195     size => {
1196         type => 'string',
1197         format => 'disk-size',
1198         format_description => 'DiskSize',
1199         description => "Disk size. This is purely informational and has no effect.",
1200         optional => 1,
1201     },
1202 };
1203
1204 my $efidisk_desc = {
1205     optional => 1,
1206     type => 'string', format => $efidisk_fmt,
1207     description => "Configure a Disk for storing EFI vars",
1208 };
1209
1210 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1211
1212 my $usb_fmt = {
1213     host => {
1214         default_key => 1,
1215         type => 'string', format => 'pve-qm-usb-device',
1216         format_description => 'HOSTUSBDEVICE|spice',
1217         description => <<EODESCR,
1218 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1219
1220  'bus-port(.port)*' (decimal numbers) or
1221  'vendor_id:product_id' (hexadeciaml numbers) or
1222  'spice'
1223
1224 You can use the 'lsusb -t' command to list existing usb devices.
1225
1226 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1227
1228 The value 'spice' can be used to add a usb redirection devices for spice.
1229 EODESCR
1230     },
1231     usb3 => {
1232         optional => 1,
1233         type => 'boolean',
1234         description => "Specifies whether if given host option is a USB3 device or port.",
1235         default => 0,
1236     },
1237 };
1238
1239 my $usbdesc = {
1240     optional => 1,
1241     type => 'string', format => $usb_fmt,
1242     description => "Configure an USB device (n is 0 to 4).",
1243 };
1244 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1245
1246 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1247 my $hostpci_fmt = {
1248     host => {
1249         default_key => 1,
1250         type => 'string',
1251         pattern => qr/$PCIRE(;$PCIRE)*/,
1252         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1253         description => <<EODESCR,
1254 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1255 of PCI virtual functions of the host. HOSTPCIID syntax is:
1256
1257 'bus:dev.func' (hexadecimal numbers)
1258
1259 You can us the 'lspci' command to list existing PCI devices.
1260 EODESCR
1261     },
1262     rombar => {
1263         type => 'boolean',
1264         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1265         optional => 1,
1266         default => 1,
1267     },
1268     romfile => {
1269         type => 'string',
1270         pattern => '[^,;]+',
1271         format_description => 'string',
1272         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1273         optional => 1,
1274     },
1275     pcie => {
1276         type => 'boolean',
1277         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1278         optional => 1,
1279         default => 0,
1280     },
1281     'x-vga' => {
1282         type => 'boolean',
1283         description =>  "Enable vfio-vga device support.",
1284         optional => 1,
1285         default => 0,
1286     },
1287     'mdev' => {
1288         type => 'string',
1289         format_description => 'string',
1290         pattern => '[^/\.:]+',
1291         optional => 1,
1292         description => <<EODESCR
1293 The type of mediated device to use.
1294 An instance of this type will be created on startup of the VM and
1295 will be cleaned up when the VM stops.
1296 EODESCR
1297     }
1298 };
1299 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1300
1301 my $hostpcidesc = {
1302         optional => 1,
1303         type => 'string', format => 'pve-qm-hostpci',
1304         description => "Map host PCI devices into guest.",
1305         verbose_description =>  <<EODESCR,
1306 Map host PCI devices into guest.
1307
1308 NOTE: This option allows direct access to host hardware. So it is no longer
1309 possible to migrate such machines - use with special care.
1310
1311 CAUTION: Experimental! User reported problems with this option.
1312 EODESCR
1313 };
1314 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1315
1316 my $serialdesc = {
1317         optional => 1,
1318         type => 'string',
1319         pattern => '(/dev/.+|socket)',
1320         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1321         verbose_description =>  <<EODESCR,
1322 Create a serial device inside the VM (n is 0 to 3), and pass through a
1323 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1324 host side (use 'qm terminal' to open a terminal connection).
1325
1326 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1327
1328 CAUTION: Experimental! User reported problems with this option.
1329 EODESCR
1330 };
1331
1332 my $paralleldesc= {
1333         optional => 1,
1334         type => 'string',
1335         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1336         description =>  "Map host parallel devices (n is 0 to 2).",
1337         verbose_description =>  <<EODESCR,
1338 Map host parallel devices (n is 0 to 2).
1339
1340 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1341
1342 CAUTION: Experimental! User reported problems with this option.
1343 EODESCR
1344 };
1345
1346 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1347     $confdesc->{"parallel$i"} = $paralleldesc;
1348 }
1349
1350 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1351     $confdesc->{"serial$i"} = $serialdesc;
1352 }
1353
1354 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1355     $confdesc->{"hostpci$i"} = $hostpcidesc;
1356 }
1357
1358 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1359     $drivename_hash->{"ide$i"} = 1;
1360     $confdesc->{"ide$i"} = $idedesc;
1361 }
1362
1363 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1364     $drivename_hash->{"sata$i"} = 1;
1365     $confdesc->{"sata$i"} = $satadesc;
1366 }
1367
1368 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1369     $drivename_hash->{"scsi$i"} = 1;
1370     $confdesc->{"scsi$i"} = $scsidesc ;
1371 }
1372
1373 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1374     $drivename_hash->{"virtio$i"} = 1;
1375     $confdesc->{"virtio$i"} = $virtiodesc;
1376 }
1377
1378 $drivename_hash->{efidisk0} = 1;
1379 $confdesc->{efidisk0} = $efidisk_desc;
1380
1381 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1382     $confdesc->{"usb$i"} = $usbdesc;
1383 }
1384
1385 my $unuseddesc = {
1386     optional => 1,
1387     type => 'string', format => 'pve-volume-id',
1388     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1389 };
1390
1391 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1392     $confdesc->{"unused$i"} = $unuseddesc;
1393 }
1394
1395 my $kvm_api_version = 0;
1396
1397 sub kvm_version {
1398     return $kvm_api_version if $kvm_api_version;
1399
1400     open my $fh, '<', '/dev/kvm'
1401         or return undef;
1402
1403     # 0xae00 => KVM_GET_API_VERSION
1404     $kvm_api_version = ioctl($fh, 0xae00, 0);
1405
1406     return $kvm_api_version;
1407 }
1408
1409 my $kvm_user_version = {};
1410 my $kvm_mtime = {};
1411
1412 sub kvm_user_version {
1413     my ($binary) = @_;
1414
1415     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1416     my $st = stat($binary);
1417
1418     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1419     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1420         $cachedmtime == $st->mtime;
1421
1422     $kvm_user_version->{$binary} = 'unknown';
1423     $kvm_mtime->{$binary} = $st->mtime;
1424
1425     my $code = sub {
1426         my $line = shift;
1427         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1428             $kvm_user_version->{$binary} = $2;
1429         }
1430     };
1431
1432     eval { run_command([$binary, '--version'], outfunc => $code); };
1433     warn $@ if $@;
1434
1435     return $kvm_user_version->{$binary};
1436
1437 }
1438
1439 sub kernel_has_vhost_net {
1440     return -c '/dev/vhost-net';
1441 }
1442
1443 sub valid_drive_names {
1444     # order is important - used to autoselect boot disk
1445     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1446             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1447             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1448             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1449             'efidisk0');
1450 }
1451
1452 sub is_valid_drivename {
1453     my $dev = shift;
1454
1455     return defined($drivename_hash->{$dev});
1456 }
1457
1458 sub option_exists {
1459     my $key = shift;
1460     return defined($confdesc->{$key});
1461 }
1462
1463 my $cdrom_path;
1464 sub get_cdrom_path {
1465
1466     return  $cdrom_path if $cdrom_path;
1467
1468     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1469     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1470     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1471 }
1472
1473 sub get_iso_path {
1474     my ($storecfg, $vmid, $cdrom) = @_;
1475
1476     if ($cdrom eq 'cdrom') {
1477         return get_cdrom_path();
1478     } elsif ($cdrom eq 'none') {
1479         return '';
1480     } elsif ($cdrom =~ m|^/|) {
1481         return $cdrom;
1482     } else {
1483         return PVE::Storage::path($storecfg, $cdrom);
1484     }
1485 }
1486
1487 # try to convert old style file names to volume IDs
1488 sub filename_to_volume_id {
1489     my ($vmid, $file, $media) = @_;
1490
1491      if (!($file eq 'none' || $file eq 'cdrom' ||
1492           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1493
1494         return undef if $file =~ m|/|;
1495
1496         if ($media && $media eq 'cdrom') {
1497             $file = "local:iso/$file";
1498         } else {
1499             $file = "local:$vmid/$file";
1500         }
1501     }
1502
1503     return $file;
1504 }
1505
1506 sub verify_media_type {
1507     my ($opt, $vtype, $media) = @_;
1508
1509     return if !$media;
1510
1511     my $etype;
1512     if ($media eq 'disk') {
1513         $etype = 'images';
1514     } elsif ($media eq 'cdrom') {
1515         $etype = 'iso';
1516     } else {
1517         die "internal error";
1518     }
1519
1520     return if ($vtype eq $etype);
1521
1522     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1523 }
1524
1525 sub cleanup_drive_path {
1526     my ($opt, $storecfg, $drive) = @_;
1527
1528     # try to convert filesystem paths to volume IDs
1529
1530     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1531         ($drive->{file} !~ m|^/dev/.+|) &&
1532         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1533         ($drive->{file} !~ m/^\d+$/)) {
1534         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1535         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1536         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1537         verify_media_type($opt, $vtype, $drive->{media});
1538         $drive->{file} = $volid;
1539     }
1540
1541     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1542 }
1543
1544 sub parse_hotplug_features {
1545     my ($data) = @_;
1546
1547     my $res = {};
1548
1549     return $res if $data eq '0';
1550
1551     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1552
1553     foreach my $feature (PVE::Tools::split_list($data)) {
1554         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1555             $res->{$1} = 1;
1556         } else {
1557             die "invalid hotplug feature '$feature'\n";
1558         }
1559     }
1560     return $res;
1561 }
1562
1563 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1564 sub pve_verify_hotplug_features {
1565     my ($value, $noerr) = @_;
1566
1567     return $value if parse_hotplug_features($value);
1568
1569     return undef if $noerr;
1570
1571     die "unable to parse hotplug option\n";
1572 }
1573
1574 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1575 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1576 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1577 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1578 #        [,iothread=on][,serial=serial][,model=model]
1579
1580 sub parse_drive {
1581     my ($key, $data) = @_;
1582
1583     my ($interface, $index);
1584
1585     if ($key =~ m/^([^\d]+)(\d+)$/) {
1586         $interface = $1;
1587         $index = $2;
1588     } else {
1589         return undef;
1590     }
1591
1592     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1593                                      : $confdesc->{$key}->{format};
1594     if (!$desc) {
1595         warn "invalid drive key: $key\n";
1596         return undef;
1597     }
1598     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1599     return undef if !$res;
1600     $res->{interface} = $interface;
1601     $res->{index} = $index;
1602
1603     my $error = 0;
1604     foreach my $opt (qw(bps bps_rd bps_wr)) {
1605         if (my $bps = defined(delete $res->{$opt})) {
1606             if (defined($res->{"m$opt"})) {
1607                 warn "both $opt and m$opt specified\n";
1608                 ++$error;
1609                 next;
1610             }
1611             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1612         }
1613     }
1614
1615     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1616     for my $requirement (
1617         [mbps_max => 'mbps'],
1618         [mbps_rd_max => 'mbps_rd'],
1619         [mbps_wr_max => 'mbps_wr'],
1620         [miops_max => 'miops'],
1621         [miops_rd_max => 'miops_rd'],
1622         [miops_wr_max => 'miops_wr'],
1623         [bps_max_length => 'mbps_max'],
1624         [bps_rd_max_length => 'mbps_rd_max'],
1625         [bps_wr_max_length => 'mbps_wr_max'],
1626         [iops_max_length => 'iops_max'],
1627         [iops_rd_max_length => 'iops_rd_max'],
1628         [iops_wr_max_length => 'iops_wr_max']) {
1629         my ($option, $requires) = @$requirement;
1630         if ($res->{$option} && !$res->{$requires}) {
1631             warn "$option requires $requires\n";
1632             ++$error;
1633         }
1634     }
1635
1636     return undef if $error;
1637
1638     return undef if $res->{mbps_rd} && $res->{mbps};
1639     return undef if $res->{mbps_wr} && $res->{mbps};
1640     return undef if $res->{iops_rd} && $res->{iops};
1641     return undef if $res->{iops_wr} && $res->{iops};
1642
1643     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1644         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1645         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1646         return undef if $res->{interface} eq 'virtio';
1647     }
1648
1649     if (my $size = $res->{size}) {
1650         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1651     }
1652
1653     return $res;
1654 }
1655
1656 sub print_drive {
1657     my ($drive) = @_;
1658     my $data = { %$drive };
1659     delete $data->{$_} for qw(index interface);
1660     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1661 }
1662
1663 sub scsi_inquiry {
1664     my($fh, $noerr) = @_;
1665
1666     my $SG_IO = 0x2285;
1667     my $SG_GET_VERSION_NUM = 0x2282;
1668
1669     my $versionbuf = "\x00" x 8;
1670     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1671     if (!$ret) {
1672         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1673         return undef;
1674     }
1675     my $version = unpack("I", $versionbuf);
1676     if ($version < 30000) {
1677         die "scsi generic interface too old\n"  if !$noerr;
1678         return undef;
1679     }
1680
1681     my $buf = "\x00" x 36;
1682     my $sensebuf = "\x00" x 8;
1683     my $cmd = pack("C x3 C x1", 0x12, 36);
1684
1685     # see /usr/include/scsi/sg.h
1686     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1687
1688     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1689                       length($sensebuf), 0, length($buf), $buf,
1690                       $cmd, $sensebuf, 6000);
1691
1692     $ret = ioctl($fh, $SG_IO, $packet);
1693     if (!$ret) {
1694         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1695         return undef;
1696     }
1697
1698     my @res = unpack($sg_io_hdr_t, $packet);
1699     if ($res[17] || $res[18]) {
1700         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1701         return undef;
1702     }
1703
1704     my $res = {};
1705     (my $byte0, my $byte1, $res->{vendor},
1706      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1707
1708     $res->{removable} = $byte1 & 128 ? 1 : 0;
1709     $res->{type} = $byte0 & 31;
1710
1711     return $res;
1712 }
1713
1714 sub path_is_scsi {
1715     my ($path) = @_;
1716
1717     my $fh = IO::File->new("+<$path") || return undef;
1718     my $res = scsi_inquiry($fh, 1);
1719     close($fh);
1720
1721     return $res;
1722 }
1723
1724 sub print_tabletdevice_full {
1725     my ($conf, $arch) = @_;
1726
1727     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1728
1729     # we use uhci for old VMs because tablet driver was buggy in older qemu
1730     my $usbbus;
1731     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1732         $usbbus = 'ehci';
1733     } else {
1734         $usbbus = 'uhci';
1735     }
1736
1737     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1738 }
1739
1740 sub print_keyboarddevice_full {
1741     my ($conf, $arch, $machine) = @_;
1742
1743     return undef if $arch ne 'aarch64';
1744
1745     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1746 }
1747
1748 sub print_drivedevice_full {
1749     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1750
1751     my $device = '';
1752     my $maxdev = 0;
1753
1754     if ($drive->{interface} eq 'virtio') {
1755         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1756         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1757         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1758     } elsif ($drive->{interface} eq 'scsi') {
1759
1760         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1761         my $unit = $drive->{index} % $maxdev;
1762         my $devicetype = 'hd';
1763         my $path = '';
1764         if (drive_is_cdrom($drive)) {
1765             $devicetype = 'cd';
1766         } else {
1767             if ($drive->{file} =~ m|^/|) {
1768                 $path = $drive->{file};
1769                 if (my $info = path_is_scsi($path)) {
1770                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1771                         $devicetype = 'block';
1772                     } elsif ($info->{type} == 1) { # tape
1773                         $devicetype = 'generic';
1774                     }
1775                 }
1776             } else {
1777                  $path = PVE::Storage::path($storecfg, $drive->{file});
1778             }
1779
1780             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1781             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1782             if ($path =~ m/^iscsi\:\/\// &&
1783                !min_version($version, 4, 1)) {
1784                 $devicetype = 'generic';
1785             }
1786         }
1787
1788         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1789             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1790         } else {
1791             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1792         }
1793
1794         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1795             $device .= ",rotation_rate=1";
1796         }
1797         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1798
1799     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1800         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1801         my $controller = int($drive->{index} / $maxdev);
1802         my $unit = $drive->{index} % $maxdev;
1803         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1804
1805         $device = "ide-$devicetype";
1806         if ($drive->{interface} eq 'ide') {
1807             $device .= ",bus=ide.$controller,unit=$unit";
1808         } else {
1809             $device .= ",bus=ahci$controller.$unit";
1810         }
1811         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1812
1813         if ($devicetype eq 'hd') {
1814             if (my $model = $drive->{model}) {
1815                 $model = URI::Escape::uri_unescape($model);
1816                 $device .= ",model=$model";
1817             }
1818             if ($drive->{ssd}) {
1819                 $device .= ",rotation_rate=1";
1820             }
1821         }
1822         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1823     } elsif ($drive->{interface} eq 'usb') {
1824         die "implement me";
1825         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1826     } else {
1827         die "unsupported interface type";
1828     }
1829
1830     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1831
1832     if (my $serial = $drive->{serial}) {
1833         $serial = URI::Escape::uri_unescape($serial);
1834         $device .= ",serial=$serial";
1835     }
1836
1837
1838     return $device;
1839 }
1840
1841 sub get_initiator_name {
1842     my $initiator;
1843
1844     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1845     while (defined(my $line = <$fh>)) {
1846         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1847         $initiator = $1;
1848         last;
1849     }
1850     $fh->close();
1851
1852     return $initiator;
1853 }
1854
1855 sub print_drive_full {
1856     my ($storecfg, $vmid, $drive) = @_;
1857
1858     my $path;
1859     my $volid = $drive->{file};
1860     my $format;
1861
1862     if (drive_is_cdrom($drive)) {
1863         $path = get_iso_path($storecfg, $vmid, $volid);
1864     } else {
1865         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1866         if ($storeid) {
1867             $path = PVE::Storage::path($storecfg, $volid);
1868             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1869             $format = qemu_img_format($scfg, $volname);
1870         } else {
1871             $path = $volid;
1872             $format = "raw";
1873         }
1874    }
1875
1876     my $opts = '';
1877     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1878     foreach my $o (@qemu_drive_options) {
1879         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1880     }
1881
1882     # snapshot only accepts on|off
1883     if (defined($drive->{snapshot})) {
1884         my $v = $drive->{snapshot} ? 'on' : 'off';
1885         $opts .= ",snapshot=$v";
1886     }
1887
1888     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1889         my ($dir, $qmpname) = @$type;
1890         if (my $v = $drive->{"mbps$dir"}) {
1891             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1892         }
1893         if (my $v = $drive->{"mbps${dir}_max"}) {
1894             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1895         }
1896         if (my $v = $drive->{"bps${dir}_max_length"}) {
1897             $opts .= ",throttling.bps$qmpname-max-length=$v";
1898         }
1899         if (my $v = $drive->{"iops${dir}"}) {
1900             $opts .= ",throttling.iops$qmpname=$v";
1901         }
1902         if (my $v = $drive->{"iops${dir}_max"}) {
1903             $opts .= ",throttling.iops$qmpname-max=$v";
1904         }
1905         if (my $v = $drive->{"iops${dir}_max_length"}) {
1906             $opts .= ",throttling.iops$qmpname-max-length=$v";
1907         }
1908     }
1909
1910     $opts .= ",format=$format" if $format && !$drive->{format};
1911
1912     my $cache_direct = 0;
1913
1914     if (my $cache = $drive->{cache}) {
1915         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1916     } elsif (!drive_is_cdrom($drive)) {
1917         $opts .= ",cache=none";
1918         $cache_direct = 1;
1919     }
1920
1921     # aio native works only with O_DIRECT
1922     if (!$drive->{aio}) {
1923         if($cache_direct) {
1924             $opts .= ",aio=native";
1925         } else {
1926             $opts .= ",aio=threads";
1927         }
1928     }
1929
1930     if (!drive_is_cdrom($drive)) {
1931         my $detectzeroes;
1932         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1933             $detectzeroes = 'off';
1934         } elsif ($drive->{discard}) {
1935             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1936         } else {
1937             # This used to be our default with discard not being specified:
1938             $detectzeroes = 'on';
1939         }
1940         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1941     }
1942
1943     my $pathinfo = $path ? "file=$path," : '';
1944
1945     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1946 }
1947
1948 sub print_netdevice_full {
1949     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1950
1951     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1952
1953     my $device = $net->{model};
1954     if ($net->{model} eq 'virtio') {
1955          $device = 'virtio-net-pci';
1956      };
1957
1958     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1959     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1960     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1961         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1962         my $vectors = $net->{queues} * 2 + 2;
1963         $tmpstr .= ",vectors=$vectors,mq=on";
1964     }
1965     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1966
1967     if ($use_old_bios_files) {
1968         my $romfile;
1969         if ($device eq 'virtio-net-pci') {
1970             $romfile = 'pxe-virtio.rom';
1971         } elsif ($device eq 'e1000') {
1972             $romfile = 'pxe-e1000.rom';
1973         } elsif ($device eq 'ne2k') {
1974             $romfile = 'pxe-ne2k_pci.rom';
1975         } elsif ($device eq 'pcnet') {
1976             $romfile = 'pxe-pcnet.rom';
1977         } elsif ($device eq 'rtl8139') {
1978             $romfile = 'pxe-rtl8139.rom';
1979         }
1980         $tmpstr .= ",romfile=$romfile" if $romfile;
1981     }
1982
1983     return $tmpstr;
1984 }
1985
1986 sub print_netdev_full {
1987     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1988
1989     my $i = '';
1990     if ($netid =~ m/^net(\d+)$/) {
1991         $i = int($1);
1992     }
1993
1994     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1995
1996     my $ifname = "tap${vmid}i$i";
1997
1998     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1999     die "interface name '$ifname' is too long (max 15 character)\n"
2000         if length($ifname) >= 16;
2001
2002     my $vhostparam = '';
2003     if (is_native($arch)) {
2004         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2005     }
2006
2007     my $vmname = $conf->{name} || "vm$vmid";
2008
2009     my $netdev = "";
2010     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2011
2012     if ($net->{bridge}) {
2013         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2014     } else {
2015         $netdev = "type=user,id=$netid,hostname=$vmname";
2016     }
2017
2018     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2019
2020     return $netdev;
2021 }
2022
2023 my $vga_map = {
2024     'cirrus' => 'cirrus-vga',
2025     'std' => 'VGA',
2026     'vmware' => 'vmware-svga',
2027     'virtio' => 'virtio-vga',
2028 };
2029
2030 sub print_vga_device {
2031     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2032
2033     my $type = $vga_map->{$vga->{type}};
2034     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2035         $type = 'virtio-gpu';
2036     }
2037     my $vgamem_mb = $vga->{memory};
2038
2039     my $max_outputs = '';
2040     if ($qxlnum) {
2041         $type = $id ? 'qxl' : 'qxl-vga';
2042
2043         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2044             # set max outputs so linux can have up to 4 qxl displays with one device
2045             if (min_version($machine_version, 4, 1)) {
2046                 $max_outputs = ",max_outputs=4";
2047             }
2048         }
2049     }
2050
2051     die "no devicetype for $vga->{type}\n" if !$type;
2052
2053     my $memory = "";
2054     if ($vgamem_mb) {
2055         if ($vga->{type} eq 'virtio') {
2056             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2057             $memory = ",max_hostmem=$bytes";
2058         } elsif ($qxlnum) {
2059             # from https://www.spice-space.org/multiple-monitors.html
2060             $memory = ",vgamem_mb=$vga->{memory}";
2061             my $ram = $vgamem_mb * 4;
2062             my $vram = $vgamem_mb * 2;
2063             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2064         } else {
2065             $memory = ",vgamem_mb=$vga->{memory}";
2066         }
2067     } elsif ($qxlnum && $id) {
2068         $memory = ",ram_size=67108864,vram_size=33554432";
2069     }
2070
2071     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2072     my $vgaid = "vga" . ($id // '');
2073     my $pciaddr;
2074
2075     if ($q35 && $vgaid eq 'vga') {
2076         # the first display uses pcie.0 bus on q35 machines
2077         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2078     } else {
2079         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2080     }
2081
2082     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2083 }
2084
2085 sub drive_is_cloudinit {
2086     my ($drive) = @_;
2087     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2088 }
2089
2090 sub drive_is_cdrom {
2091     my ($drive, $exclude_cloudinit) = @_;
2092
2093     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2094
2095     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2096
2097 }
2098
2099 sub parse_number_sets {
2100     my ($set) = @_;
2101     my $res = [];
2102     foreach my $part (split(/;/, $set)) {
2103         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2104             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2105             push @$res, [ $1, $2 ];
2106         } else {
2107             die "invalid range: $part\n";
2108         }
2109     }
2110     return $res;
2111 }
2112
2113 sub parse_numa {
2114     my ($data) = @_;
2115
2116     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2117     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2118     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2119     return $res;
2120 }
2121
2122 sub parse_hostpci {
2123     my ($value) = @_;
2124
2125     return undef if !$value;
2126
2127     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2128
2129     my @idlist = split(/;/, $res->{host});
2130     delete $res->{host};
2131     foreach my $id (@idlist) {
2132         my $devs = PVE::SysFSTools::lspci($id);
2133         die "no PCI device found for '$id'\n" if !scalar(@$devs);
2134         push @{$res->{pciid}}, @$devs;
2135     }
2136     return $res;
2137 }
2138
2139 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2140 sub parse_net {
2141     my ($data) = @_;
2142
2143     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2144     if ($@) {
2145         warn $@;
2146         return undef;
2147     }
2148     if (!defined($res->{macaddr})) {
2149         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2150         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2151     }
2152     return $res;
2153 }
2154
2155 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2156 sub parse_ipconfig {
2157     my ($data) = @_;
2158
2159     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2160     if ($@) {
2161         warn $@;
2162         return undef;
2163     }
2164
2165     if ($res->{gw} && !$res->{ip}) {
2166         warn 'gateway specified without specifying an IP address';
2167         return undef;
2168     }
2169     if ($res->{gw6} && !$res->{ip6}) {
2170         warn 'IPv6 gateway specified without specifying an IPv6 address';
2171         return undef;
2172     }
2173     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2174         warn 'gateway specified together with DHCP';
2175         return undef;
2176     }
2177     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2178         # gw6 + auto/dhcp
2179         warn "IPv6 gateway specified together with $res->{ip6} address";
2180         return undef;
2181     }
2182
2183     if (!$res->{ip} && !$res->{ip6}) {
2184         return { ip => 'dhcp', ip6 => 'dhcp' };
2185     }
2186
2187     return $res;
2188 }
2189
2190 sub print_net {
2191     my $net = shift;
2192
2193     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2194 }
2195
2196 sub add_random_macs {
2197     my ($settings) = @_;
2198
2199     foreach my $opt (keys %$settings) {
2200         next if $opt !~ m/^net(\d+)$/;
2201         my $net = parse_net($settings->{$opt});
2202         next if !$net;
2203         $settings->{$opt} = print_net($net);
2204     }
2205 }
2206
2207 sub vm_is_volid_owner {
2208     my ($storecfg, $vmid, $volid) = @_;
2209
2210     if ($volid !~  m|^/|) {
2211         my ($path, $owner);
2212         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2213         if ($owner && ($owner == $vmid)) {
2214             return 1;
2215         }
2216     }
2217
2218     return undef;
2219 }
2220
2221 sub vmconfig_register_unused_drive {
2222     my ($storecfg, $vmid, $conf, $drive) = @_;
2223
2224     if (drive_is_cloudinit($drive)) {
2225         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2226         warn $@ if $@;
2227     } elsif (!drive_is_cdrom($drive)) {
2228         my $volid = $drive->{file};
2229         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2230             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2231         }
2232     }
2233 }
2234
2235 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2236 my $smbios1_fmt = {
2237     uuid => {
2238         type => 'string',
2239         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2240         format_description => 'UUID',
2241         description => "Set SMBIOS1 UUID.",
2242         optional => 1,
2243     },
2244     version => {
2245         type => 'string',
2246         pattern => '[A-Za-z0-9+\/]+={0,2}',
2247         format_description => 'Base64 encoded string',
2248         description => "Set SMBIOS1 version.",
2249         optional => 1,
2250     },
2251     serial => {
2252         type => 'string',
2253         pattern => '[A-Za-z0-9+\/]+={0,2}',
2254         format_description => 'Base64 encoded string',
2255         description => "Set SMBIOS1 serial number.",
2256         optional => 1,
2257     },
2258     manufacturer => {
2259         type => 'string',
2260         pattern => '[A-Za-z0-9+\/]+={0,2}',
2261         format_description => 'Base64 encoded string',
2262         description => "Set SMBIOS1 manufacturer.",
2263         optional => 1,
2264     },
2265     product => {
2266         type => 'string',
2267         pattern => '[A-Za-z0-9+\/]+={0,2}',
2268         format_description => 'Base64 encoded string',
2269         description => "Set SMBIOS1 product ID.",
2270         optional => 1,
2271     },
2272     sku => {
2273         type => 'string',
2274         pattern => '[A-Za-z0-9+\/]+={0,2}',
2275         format_description => 'Base64 encoded string',
2276         description => "Set SMBIOS1 SKU string.",
2277         optional => 1,
2278     },
2279     family => {
2280         type => 'string',
2281         pattern => '[A-Za-z0-9+\/]+={0,2}',
2282         format_description => 'Base64 encoded string',
2283         description => "Set SMBIOS1 family string.",
2284         optional => 1,
2285     },
2286     base64 => {
2287         type => 'boolean',
2288         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2289         optional => 1,
2290     },
2291 };
2292
2293 sub parse_smbios1 {
2294     my ($data) = @_;
2295
2296     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2297     warn $@ if $@;
2298     return $res;
2299 }
2300
2301 sub print_smbios1 {
2302     my ($smbios1) = @_;
2303     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2304 }
2305
2306 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2307
2308 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2309 sub verify_bootdisk {
2310     my ($value, $noerr) = @_;
2311
2312     return $value if is_valid_drivename($value);
2313
2314     return undef if $noerr;
2315
2316     die "invalid boot disk '$value'\n";
2317 }
2318
2319 sub parse_watchdog {
2320     my ($value) = @_;
2321
2322     return undef if !$value;
2323
2324     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2325     warn $@ if $@;
2326     return $res;
2327 }
2328
2329 sub parse_guest_agent {
2330     my ($value) = @_;
2331
2332     return {} if !defined($value->{agent});
2333
2334     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2335     warn $@ if $@;
2336
2337     # if the agent is disabled ignore the other potentially set properties
2338     return {} if !$res->{enabled};
2339     return $res;
2340 }
2341
2342 sub parse_vga {
2343     my ($value) = @_;
2344
2345     return {} if !$value;
2346     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2347     warn $@ if $@;
2348     return $res;
2349 }
2350
2351 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2352 sub verify_usb_device {
2353     my ($value, $noerr) = @_;
2354
2355     return $value if parse_usb_device($value);
2356
2357     return undef if $noerr;
2358
2359     die "unable to parse usb device\n";
2360 }
2361
2362 # add JSON properties for create and set function
2363 sub json_config_properties {
2364     my $prop = shift;
2365
2366     foreach my $opt (keys %$confdesc) {
2367         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2368         $prop->{$opt} = $confdesc->{$opt};
2369     }
2370
2371     return $prop;
2372 }
2373
2374 # return copy of $confdesc_cloudinit to generate documentation
2375 sub cloudinit_config_properties {
2376
2377     return dclone($confdesc_cloudinit);
2378 }
2379
2380 sub check_type {
2381     my ($key, $value) = @_;
2382
2383     die "unknown setting '$key'\n" if !$confdesc->{$key};
2384
2385     my $type = $confdesc->{$key}->{type};
2386
2387     if (!defined($value)) {
2388         die "got undefined value\n";
2389     }
2390
2391     if ($value =~ m/[\n\r]/) {
2392         die "property contains a line feed\n";
2393     }
2394
2395     if ($type eq 'boolean') {
2396         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2397         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2398         die "type check ('boolean') failed - got '$value'\n";
2399     } elsif ($type eq 'integer') {
2400         return int($1) if $value =~ m/^(\d+)$/;
2401         die "type check ('integer') failed - got '$value'\n";
2402     } elsif ($type eq 'number') {
2403         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2404         die "type check ('number') failed - got '$value'\n";
2405     } elsif ($type eq 'string') {
2406         if (my $fmt = $confdesc->{$key}->{format}) {
2407             PVE::JSONSchema::check_format($fmt, $value);
2408             return $value;
2409         }
2410         $value =~ s/^\"(.*)\"$/$1/;
2411         return $value;
2412     } else {
2413         die "internal error"
2414     }
2415 }
2416
2417 sub destroy_vm {
2418     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2419
2420     my $conf = PVE::QemuConfig->load_config($vmid);
2421
2422     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2423
2424     if ($conf->{template}) {
2425         # check if any base image is still used by a linked clone
2426         foreach_drive($conf, sub {
2427                 my ($ds, $drive) = @_;
2428                 return if drive_is_cdrom($drive);
2429
2430                 my $volid = $drive->{file};
2431                 return if !$volid || $volid =~ m|^/|;
2432
2433                 die "base volume '$volid' is still in use by linked cloned\n"
2434                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2435
2436         });
2437     }
2438
2439     # only remove disks owned by this VM
2440     foreach_drive($conf, sub {
2441         my ($ds, $drive) = @_;
2442         return if drive_is_cdrom($drive, 1);
2443
2444         my $volid = $drive->{file};
2445         return if !$volid || $volid =~ m|^/|;
2446
2447         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2448         return if !$path || !$owner || ($owner != $vmid);
2449
2450         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2451         warn "Could not remove disk '$volid', check manually: $@" if $@;
2452     });
2453
2454     # also remove unused disk
2455     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2456     PVE::Storage::foreach_volid($vmdisks, sub {
2457         my ($volid, $sid, $volname, $d) = @_;
2458         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2459         warn $@ if $@;
2460     });
2461
2462     if (defined $replacement_conf) {
2463         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2464     } else {
2465         PVE::QemuConfig->destroy_config($vmid);
2466     }
2467 }
2468
2469 sub parse_vm_config {
2470     my ($filename, $raw) = @_;
2471
2472     return undef if !defined($raw);
2473
2474     my $res = {
2475         digest => Digest::SHA::sha1_hex($raw),
2476         snapshots => {},
2477         pending => {},
2478     };
2479
2480     $filename =~ m|/qemu-server/(\d+)\.conf$|
2481         || die "got strange filename '$filename'";
2482
2483     my $vmid = $1;
2484
2485     my $conf = $res;
2486     my $descr;
2487     my $section = '';
2488
2489     my @lines = split(/\n/, $raw);
2490     foreach my $line (@lines) {
2491         next if $line =~ m/^\s*$/;
2492
2493         if ($line =~ m/^\[PENDING\]\s*$/i) {
2494             $section = 'pending';
2495             if (defined($descr)) {
2496                 $descr =~ s/\s+$//;
2497                 $conf->{description} = $descr;
2498             }
2499             $descr = undef;
2500             $conf = $res->{$section} = {};
2501             next;
2502
2503         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2504             $section = $1;
2505             if (defined($descr)) {
2506                 $descr =~ s/\s+$//;
2507                 $conf->{description} = $descr;
2508             }
2509             $descr = undef;
2510             $conf = $res->{snapshots}->{$section} = {};
2511             next;
2512         }
2513
2514         if ($line =~ m/^\#(.*)\s*$/) {
2515             $descr = '' if !defined($descr);
2516             $descr .= PVE::Tools::decode_text($1) . "\n";
2517             next;
2518         }
2519
2520         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2521             $descr = '' if !defined($descr);
2522             $descr .= PVE::Tools::decode_text($2);
2523         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2524             $conf->{snapstate} = $1;
2525         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2526             my $key = $1;
2527             my $value = $2;
2528             $conf->{$key} = $value;
2529         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2530             my $value = $1;
2531             if ($section eq 'pending') {
2532                 $conf->{delete} = $value; # we parse this later
2533             } else {
2534                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2535             }
2536         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2537             my $key = $1;
2538             my $value = $2;
2539             eval { $value = check_type($key, $value); };
2540             if ($@) {
2541                 warn "vm $vmid - unable to parse value of '$key' - $@";
2542             } else {
2543                 $key = 'ide2' if $key eq 'cdrom';
2544                 my $fmt = $confdesc->{$key}->{format};
2545                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2546                     my $v = parse_drive($key, $value);
2547                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2548                         $v->{file} = $volid;
2549                         $value = print_drive($v);
2550                     } else {
2551                         warn "vm $vmid - unable to parse value of '$key'\n";
2552                         next;
2553                     }
2554                 }
2555
2556                 $conf->{$key} = $value;
2557             }
2558         }
2559     }
2560
2561     if (defined($descr)) {
2562         $descr =~ s/\s+$//;
2563         $conf->{description} = $descr;
2564     }
2565     delete $res->{snapstate}; # just to be sure
2566
2567     return $res;
2568 }
2569
2570 sub write_vm_config {
2571     my ($filename, $conf) = @_;
2572
2573     delete $conf->{snapstate}; # just to be sure
2574
2575     if ($conf->{cdrom}) {
2576         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2577         $conf->{ide2} = $conf->{cdrom};
2578         delete $conf->{cdrom};
2579     }
2580
2581     # we do not use 'smp' any longer
2582     if ($conf->{sockets}) {
2583         delete $conf->{smp};
2584     } elsif ($conf->{smp}) {
2585         $conf->{sockets} = $conf->{smp};
2586         delete $conf->{cores};
2587         delete $conf->{smp};
2588     }
2589
2590     my $used_volids = {};
2591
2592     my $cleanup_config = sub {
2593         my ($cref, $pending, $snapname) = @_;
2594
2595         foreach my $key (keys %$cref) {
2596             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2597                 $key eq 'snapstate' || $key eq 'pending';
2598             my $value = $cref->{$key};
2599             if ($key eq 'delete') {
2600                 die "propertry 'delete' is only allowed in [PENDING]\n"
2601                     if !$pending;
2602                 # fixme: check syntax?
2603                 next;
2604             }
2605             eval { $value = check_type($key, $value); };
2606             die "unable to parse value of '$key' - $@" if $@;
2607
2608             $cref->{$key} = $value;
2609
2610             if (!$snapname && is_valid_drivename($key)) {
2611                 my $drive = parse_drive($key, $value);
2612                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2613             }
2614         }
2615     };
2616
2617     &$cleanup_config($conf);
2618
2619     &$cleanup_config($conf->{pending}, 1);
2620
2621     foreach my $snapname (keys %{$conf->{snapshots}}) {
2622         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2623         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2624     }
2625
2626     # remove 'unusedX' settings if we re-add a volume
2627     foreach my $key (keys %$conf) {
2628         my $value = $conf->{$key};
2629         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2630             delete $conf->{$key};
2631         }
2632     }
2633
2634     my $generate_raw_config = sub {
2635         my ($conf, $pending) = @_;
2636
2637         my $raw = '';
2638
2639         # add description as comment to top of file
2640         if (defined(my $descr = $conf->{description})) {
2641             if ($descr) {
2642                 foreach my $cl (split(/\n/, $descr)) {
2643                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2644                 }
2645             } else {
2646                 $raw .= "#\n" if $pending;
2647             }
2648         }
2649
2650         foreach my $key (sort keys %$conf) {
2651             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2652             $raw .= "$key: $conf->{$key}\n";
2653         }
2654         return $raw;
2655     };
2656
2657     my $raw = &$generate_raw_config($conf);
2658
2659     if (scalar(keys %{$conf->{pending}})){
2660         $raw .= "\n[PENDING]\n";
2661         $raw .= &$generate_raw_config($conf->{pending}, 1);
2662     }
2663
2664     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2665         $raw .= "\n[$snapname]\n";
2666         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2667     }
2668
2669     return $raw;
2670 }
2671
2672 sub load_defaults {
2673
2674     my $res = {};
2675
2676     # we use static defaults from our JSON schema configuration
2677     foreach my $key (keys %$confdesc) {
2678         if (defined(my $default = $confdesc->{$key}->{default})) {
2679             $res->{$key} = $default;
2680         }
2681     }
2682
2683     return $res;
2684 }
2685
2686 sub config_list {
2687     my $vmlist = PVE::Cluster::get_vmlist();
2688     my $res = {};
2689     return $res if !$vmlist || !$vmlist->{ids};
2690     my $ids = $vmlist->{ids};
2691     my $nodename = nodename();
2692
2693     foreach my $vmid (keys %$ids) {
2694         my $d = $ids->{$vmid};
2695         next if !$d->{node} || $d->{node} ne $nodename;
2696         next if !$d->{type} || $d->{type} ne 'qemu';
2697         $res->{$vmid}->{exists} = 1;
2698     }
2699     return $res;
2700 }
2701
2702 # test if VM uses local resources (to prevent migration)
2703 sub check_local_resources {
2704     my ($conf, $noerr) = @_;
2705
2706     my @loc_res = ();
2707
2708     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2709     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2710
2711     push @loc_res, "ivshmem" if $conf->{ivshmem};
2712
2713     foreach my $k (keys %$conf) {
2714         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2715         # sockets are safe: they will recreated be on the target side post-migrate
2716         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2717         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2718     }
2719
2720     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2721
2722     return \@loc_res;
2723 }
2724
2725 # check if used storages are available on all nodes (use by migrate)
2726 sub check_storage_availability {
2727     my ($storecfg, $conf, $node) = @_;
2728
2729     foreach_drive($conf, sub {
2730         my ($ds, $drive) = @_;
2731
2732         my $volid = $drive->{file};
2733         return if !$volid;
2734
2735         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2736         return if !$sid;
2737
2738         # check if storage is available on both nodes
2739         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2740         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2741    });
2742 }
2743
2744 # list nodes where all VM images are available (used by has_feature API)
2745 sub shared_nodes {
2746     my ($conf, $storecfg) = @_;
2747
2748     my $nodelist = PVE::Cluster::get_nodelist();
2749     my $nodehash = { map { $_ => 1 } @$nodelist };
2750     my $nodename = nodename();
2751
2752     foreach_drive($conf, sub {
2753         my ($ds, $drive) = @_;
2754
2755         my $volid = $drive->{file};
2756         return if !$volid;
2757
2758         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2759         if ($storeid) {
2760             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2761             if ($scfg->{disable}) {
2762                 $nodehash = {};
2763             } elsif (my $avail = $scfg->{nodes}) {
2764                 foreach my $node (keys %$nodehash) {
2765                     delete $nodehash->{$node} if !$avail->{$node};
2766                 }
2767             } elsif (!$scfg->{shared}) {
2768                 foreach my $node (keys %$nodehash) {
2769                     delete $nodehash->{$node} if $node ne $nodename
2770                 }
2771             }
2772         }
2773     });
2774
2775     return $nodehash
2776 }
2777
2778 sub check_local_storage_availability {
2779     my ($conf, $storecfg) = @_;
2780
2781     my $nodelist = PVE::Cluster::get_nodelist();
2782     my $nodehash = { map { $_ => {} } @$nodelist };
2783
2784     foreach_drive($conf, sub {
2785         my ($ds, $drive) = @_;
2786
2787         my $volid = $drive->{file};
2788         return if !$volid;
2789
2790         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2791         if ($storeid) {
2792             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2793
2794             if ($scfg->{disable}) {
2795                 foreach my $node (keys %$nodehash) {
2796                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2797                 }
2798             } elsif (my $avail = $scfg->{nodes}) {
2799                 foreach my $node (keys %$nodehash) {
2800                     if (!$avail->{$node}) {
2801                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2802                     }
2803                 }
2804             }
2805         }
2806     });
2807
2808     foreach my $node (values %$nodehash) {
2809         if (my $unavail = $node->{unavailable_storages}) {
2810             $node->{unavailable_storages} = [ sort keys %$unavail ];
2811         }
2812     }
2813
2814     return $nodehash
2815 }
2816
2817 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2818 sub check_running {
2819     my ($vmid, $nocheck, $node) = @_;
2820
2821     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2822     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2823 }
2824
2825 sub vzlist {
2826
2827     my $vzlist = config_list();
2828
2829     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2830
2831     while (defined(my $de = $fd->read)) {
2832         next if $de !~ m/^(\d+)\.pid$/;
2833         my $vmid = $1;
2834         next if !defined($vzlist->{$vmid});
2835         if (my $pid = check_running($vmid)) {
2836             $vzlist->{$vmid}->{pid} = $pid;
2837         }
2838     }
2839
2840     return $vzlist;
2841 }
2842
2843 sub disksize {
2844     my ($storecfg, $conf) = @_;
2845
2846     my $bootdisk = $conf->{bootdisk};
2847     return undef if !$bootdisk;
2848     return undef if !is_valid_drivename($bootdisk);
2849
2850     return undef if !$conf->{$bootdisk};
2851
2852     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2853     return undef if !defined($drive);
2854
2855     return undef if drive_is_cdrom($drive);
2856
2857     my $volid = $drive->{file};
2858     return undef if !$volid;
2859
2860     return $drive->{size};
2861 }
2862
2863 our $vmstatus_return_properties = {
2864     vmid => get_standard_option('pve-vmid'),
2865     status => {
2866         description => "Qemu process status.",
2867         type => 'string',
2868         enum => ['stopped', 'running'],
2869     },
2870     maxmem => {
2871         description => "Maximum memory in bytes.",
2872         type => 'integer',
2873         optional => 1,
2874         renderer => 'bytes',
2875     },
2876     maxdisk => {
2877         description => "Root disk size in bytes.",
2878         type => 'integer',
2879         optional => 1,
2880         renderer => 'bytes',
2881     },
2882     name => {
2883         description => "VM name.",
2884         type => 'string',
2885         optional => 1,
2886     },
2887     qmpstatus => {
2888         description => "Qemu QMP agent status.",
2889         type => 'string',
2890         optional => 1,
2891     },
2892     pid => {
2893         description => "PID of running qemu process.",
2894         type => 'integer',
2895         optional => 1,
2896     },
2897     uptime => {
2898         description => "Uptime.",
2899         type => 'integer',
2900         optional => 1,
2901         renderer => 'duration',
2902     },
2903     cpus => {
2904         description => "Maximum usable CPUs.",
2905         type => 'number',
2906         optional => 1,
2907     },
2908     lock => {
2909         description => "The current config lock, if any.",
2910         type => 'string',
2911         optional => 1,
2912     },
2913     tags => {
2914         description  => "The current configured tags, if any",
2915         type => 'string',
2916         optional => 1,
2917     },
2918 };
2919
2920 my $last_proc_pid_stat;
2921
2922 # get VM status information
2923 # This must be fast and should not block ($full == false)
2924 # We only query KVM using QMP if $full == true (this can be slow)
2925 sub vmstatus {
2926     my ($opt_vmid, $full) = @_;
2927
2928     my $res = {};
2929
2930     my $storecfg = PVE::Storage::config();
2931
2932     my $list = vzlist();
2933     my $defaults = load_defaults();
2934
2935     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2936
2937     my $cpucount = $cpuinfo->{cpus} || 1;
2938
2939     foreach my $vmid (keys %$list) {
2940         next if $opt_vmid && ($vmid ne $opt_vmid);
2941
2942         my $conf = PVE::QemuConfig->load_config($vmid);
2943
2944         my $d = { vmid => $vmid };
2945         $d->{pid} = $list->{$vmid}->{pid};
2946
2947         # fixme: better status?
2948         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2949
2950         my $size = disksize($storecfg, $conf);
2951         if (defined($size)) {
2952             $d->{disk} = 0; # no info available
2953             $d->{maxdisk} = $size;
2954         } else {
2955             $d->{disk} = 0;
2956             $d->{maxdisk} = 0;
2957         }
2958
2959         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2960             * ($conf->{cores} || $defaults->{cores});
2961         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2962         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2963
2964         $d->{name} = $conf->{name} || "VM $vmid";
2965         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2966             : $defaults->{memory}*(1024*1024);
2967
2968         if ($conf->{balloon}) {
2969             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2970             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2971                 : $defaults->{shares};
2972         }
2973
2974         $d->{uptime} = 0;
2975         $d->{cpu} = 0;
2976         $d->{mem} = 0;
2977
2978         $d->{netout} = 0;
2979         $d->{netin} = 0;
2980
2981         $d->{diskread} = 0;
2982         $d->{diskwrite} = 0;
2983
2984         $d->{template} = PVE::QemuConfig->is_template($conf);
2985
2986         $d->{serial} = 1 if conf_has_serial($conf);
2987         $d->{lock} = $conf->{lock} if $conf->{lock};
2988         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2989
2990         $res->{$vmid} = $d;
2991     }
2992
2993     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2994     foreach my $dev (keys %$netdev) {
2995         next if $dev !~ m/^tap([1-9]\d*)i/;
2996         my $vmid = $1;
2997         my $d = $res->{$vmid};
2998         next if !$d;
2999
3000         $d->{netout} += $netdev->{$dev}->{receive};
3001         $d->{netin} += $netdev->{$dev}->{transmit};
3002
3003         if ($full) {
3004             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3005             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3006         }
3007
3008     }
3009
3010     my $ctime = gettimeofday;
3011
3012     foreach my $vmid (keys %$list) {
3013
3014         my $d = $res->{$vmid};
3015         my $pid = $d->{pid};
3016         next if !$pid;
3017
3018         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3019         next if !$pstat; # not running
3020
3021         my $used = $pstat->{utime} + $pstat->{stime};
3022
3023         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3024
3025         if ($pstat->{vsize}) {
3026             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3027         }
3028
3029         my $old = $last_proc_pid_stat->{$pid};
3030         if (!$old) {
3031             $last_proc_pid_stat->{$pid} = {
3032                 time => $ctime,
3033                 used => $used,
3034                 cpu => 0,
3035             };
3036             next;
3037         }
3038
3039         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3040
3041         if ($dtime > 1000) {
3042             my $dutime = $used -  $old->{used};
3043
3044             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3045             $last_proc_pid_stat->{$pid} = {
3046                 time => $ctime,
3047                 used => $used,
3048                 cpu => $d->{cpu},
3049             };
3050         } else {
3051             $d->{cpu} = $old->{cpu};
3052         }
3053     }
3054
3055     return $res if !$full;
3056
3057     my $qmpclient = PVE::QMPClient->new();
3058
3059     my $ballooncb = sub {
3060         my ($vmid, $resp) = @_;
3061
3062         my $info = $resp->{'return'};
3063         return if !$info->{max_mem};
3064
3065         my $d = $res->{$vmid};
3066
3067         # use memory assigned to VM
3068         $d->{maxmem} = $info->{max_mem};
3069         $d->{balloon} = $info->{actual};
3070
3071         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3072             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3073             $d->{freemem} = $info->{free_mem};
3074         }
3075
3076         $d->{ballooninfo} = $info;
3077     };
3078
3079     my $blockstatscb = sub {
3080         my ($vmid, $resp) = @_;
3081         my $data = $resp->{'return'} || [];
3082         my $totalrdbytes = 0;
3083         my $totalwrbytes = 0;
3084
3085         for my $blockstat (@$data) {
3086             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3087             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3088
3089             $blockstat->{device} =~ s/drive-//;
3090             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3091         }
3092         $res->{$vmid}->{diskread} = $totalrdbytes;
3093         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3094     };
3095
3096     my $statuscb = sub {
3097         my ($vmid, $resp) = @_;
3098
3099         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3100         # this fails if ballon driver is not loaded, so this must be
3101         # the last commnand (following command are aborted if this fails).
3102         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3103
3104         my $status = 'unknown';
3105         if (!defined($status = $resp->{'return'}->{status})) {
3106             warn "unable to get VM status\n";
3107             return;
3108         }
3109
3110         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3111     };
3112
3113     foreach my $vmid (keys %$list) {
3114         next if $opt_vmid && ($vmid ne $opt_vmid);
3115         next if !$res->{$vmid}->{pid}; # not running
3116         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3117     }
3118
3119     $qmpclient->queue_execute(undef, 2);
3120
3121     foreach my $vmid (keys %$list) {
3122         next if $opt_vmid && ($vmid ne $opt_vmid);
3123         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3124     }
3125
3126     return $res;
3127 }
3128
3129 sub foreach_drive {
3130     my ($conf, $func, @param) = @_;
3131
3132     foreach my $ds (valid_drive_names()) {
3133         next if !defined($conf->{$ds});
3134
3135         my $drive = parse_drive($ds, $conf->{$ds});
3136         next if !$drive;
3137
3138         &$func($ds, $drive, @param);
3139     }
3140 }
3141
3142 sub foreach_volid {
3143     my ($conf, $func, @param) = @_;
3144
3145     my $volhash = {};
3146
3147     my $test_volid = sub {
3148         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3149
3150         return if !$volid;
3151
3152         $volhash->{$volid}->{cdrom} //= 1;
3153         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3154
3155         $volhash->{$volid}->{replicate} //= 0;
3156         $volhash->{$volid}->{replicate} = 1 if $replicate;
3157
3158         $volhash->{$volid}->{shared} //= 0;
3159         $volhash->{$volid}->{shared} = 1 if $shared;
3160
3161         $volhash->{$volid}->{referenced_in_config} //= 0;
3162         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3163
3164         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3165             if defined($snapname);
3166         $volhash->{$volid}->{size} = $size if $size;
3167     };
3168
3169     foreach_drive($conf, sub {
3170         my ($ds, $drive) = @_;
3171         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3172     });
3173
3174     foreach my $snapname (keys %{$conf->{snapshots}}) {
3175         my $snap = $conf->{snapshots}->{$snapname};
3176         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3177         foreach_drive($snap, sub {
3178             my ($ds, $drive) = @_;
3179             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3180         });
3181     }
3182
3183     foreach my $volid (keys %$volhash) {
3184         &$func($volid, $volhash->{$volid}, @param);
3185     }
3186 }
3187
3188 sub conf_has_serial {
3189     my ($conf) = @_;
3190
3191     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3192         if ($conf->{"serial$i"}) {
3193             return 1;
3194         }
3195     }
3196
3197     return 0;
3198 }
3199
3200 sub conf_has_audio {
3201     my ($conf, $id) = @_;
3202
3203     $id //= 0;
3204     my $audio = $conf->{"audio$id"};
3205     return undef if !defined($audio);
3206
3207     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3208     my $audiodriver = $audioproperties->{driver} // 'spice';
3209
3210     return {
3211         dev => $audioproperties->{device},
3212         dev_id => "audiodev$id",
3213         backend => $audiodriver,
3214         backend_id => "$audiodriver-backend${id}",
3215     };
3216 }
3217
3218 sub vga_conf_has_spice {
3219     my ($vga) = @_;
3220
3221     my $vgaconf = parse_vga($vga);
3222     my $vgatype = $vgaconf->{type};
3223     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3224
3225     return $1 || 1;
3226 }
3227
3228 sub is_native($) {
3229     my ($arch) = @_;
3230     return get_host_arch() eq $arch;
3231 }
3232
3233 sub get_vm_arch {
3234     my ($conf) = @_;
3235     return $conf->{arch} // get_host_arch();
3236 }
3237
3238 my $default_machines = {
3239     x86_64 => 'pc',
3240     aarch64 => 'virt',
3241 };
3242
3243 sub get_vm_machine {
3244     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3245
3246     my $machine = $forcemachine || $conf->{machine};
3247
3248     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3249         $arch //= 'x86_64';
3250         $machine ||= $default_machines->{$arch};
3251         if ($add_pve_version) {
3252             $kvmversion //= kvm_user_version();
3253             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3254             $machine .= "+pve$pvever";
3255         }
3256     }
3257
3258     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
3259         # for version-pinned machines that do not include a pve-version (e.g.
3260         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3261         $machine .= '+pve0';
3262     }
3263
3264     return $machine;
3265 }
3266
3267 sub get_ovmf_files($) {
3268     my ($arch) = @_;
3269
3270     my $ovmf = $OVMF->{$arch}
3271         or die "no OVMF images known for architecture '$arch'\n";
3272
3273     return @$ovmf;
3274 }
3275
3276 my $Arch2Qemu = {
3277     aarch64 => '/usr/bin/qemu-system-aarch64',
3278     x86_64 => '/usr/bin/qemu-system-x86_64',
3279 };
3280 sub get_command_for_arch($) {
3281     my ($arch) = @_;
3282     return '/usr/bin/kvm' if is_native($arch);
3283
3284     my $cmd = $Arch2Qemu->{$arch}
3285         or die "don't know how to emulate architecture '$arch'\n";
3286     return $cmd;
3287 }
3288
3289 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3290 # to use in a QEMU command line (-cpu element), first array_intersect the result
3291 # of query_supported_ with query_understood_. This is necessary because:
3292 #
3293 # a) query_understood_ returns flags the host cannot use and
3294 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3295 #    flags, but CPU settings - with most of them being flags. Those settings
3296 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3297 #
3298 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3299 # expensive. If you need the value returned from this, you can get it much
3300 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3301 # $accel being 'kvm' or 'tcg'.
3302 #
3303 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3304 # changes, automatically populating pmxcfs.
3305 #
3306 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3307 # since kvm and tcg machines support different flags
3308 #
3309 sub query_supported_cpu_flags {
3310     my ($arch) = @_;
3311
3312     $arch //= get_host_arch();
3313     my $default_machine = $default_machines->{$arch};
3314
3315     my $flags = {};
3316
3317     # FIXME: Once this is merged, the code below should work for ARM as well:
3318     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3319     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3320         $arch eq "aarch64";
3321
3322     my $kvm_supported = defined(kvm_version());
3323     my $qemu_cmd = get_command_for_arch($arch);
3324     my $fakevmid = -1;
3325     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3326
3327     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3328     my $query_supported_run_qemu = sub {
3329         my ($kvm) = @_;
3330
3331         my $flags = {};
3332         my $cmd = [
3333             $qemu_cmd,
3334             '-machine', $default_machine,
3335             '-display', 'none',
3336             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3337             '-mon', 'chardev=qmp,mode=control',
3338             '-pidfile', $pidfile,
3339             '-S', '-daemonize'
3340         ];
3341
3342         if (!$kvm) {
3343             push @$cmd, '-accel', 'tcg';
3344         }
3345
3346         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3347         die "QEMU flag querying VM exited with code " . $rc if $rc;
3348
3349         eval {
3350             my $cmd_result = mon_cmd(
3351                 $fakevmid,
3352                 'query-cpu-model-expansion',
3353                 type => 'full',
3354                 model => { name => 'host' }
3355             );
3356
3357             my $props = $cmd_result->{model}->{props};
3358             foreach my $prop (keys %$props) {
3359                 next if $props->{$prop} ne '1';
3360                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3361                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3362                 # We only keep those with underscores, to match /proc/cpuinfo
3363                 $prop =~ s/\.|-/_/g;
3364                 $flags->{$prop} = 1;
3365             }
3366         };
3367         my $err = $@;
3368
3369         # force stop with 10 sec timeout and 'nocheck'
3370         # always stop, even if QMP failed
3371         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3372
3373         die $err if $err;
3374
3375         return [ sort keys %$flags ];
3376     };
3377
3378     # We need to query QEMU twice, since KVM and TCG have different supported flags
3379     PVE::QemuConfig->lock_config($fakevmid, sub {
3380         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3381         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3382
3383         if ($kvm_supported) {
3384             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3385             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3386         }
3387     });
3388
3389     return $flags;
3390 }
3391
3392 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3393 my $understood_cpu_flag_dir = "/usr/share/kvm";
3394 sub query_understood_cpu_flags {
3395     my $arch = get_host_arch();
3396     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3397
3398     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3399         if ! -e $filepath;
3400
3401     my $raw = file_get_contents($filepath);
3402     $raw =~ s/^\s+|\s+$//g;
3403     my @flags = split(/\s+/, $raw);
3404
3405     return \@flags;
3406 }
3407
3408 sub config_to_command {
3409     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3410
3411     my $cmd = [];
3412     my $globalFlags = [];
3413     my $machineFlags = [];
3414     my $rtcFlags = [];
3415     my $devices = [];
3416     my $pciaddr = '';
3417     my $bridges = {};
3418     my $ostype = $conf->{ostype};
3419     my $winversion = windows_version($ostype);
3420     my $kvm = $conf->{kvm};
3421     my $nodename = nodename();
3422
3423     my $arch = get_vm_arch($conf);
3424     my $kvm_binary = get_command_for_arch($arch);
3425     my $kvmver = kvm_user_version($kvm_binary);
3426
3427     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3428         $kvmver //= "undefined";
3429         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3430     }
3431
3432     my $add_pve_version = min_version($kvmver, 4, 1);
3433
3434     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3435     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3436     $kvm //= 1 if is_native($arch);
3437
3438     $machine_version =~ m/(\d+)\.(\d+)/;
3439     my ($machine_major, $machine_minor) = ($1, $2);
3440     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3441         if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);
3442
3443     if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3444         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3445         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
3446           . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
3447     }
3448
3449     # if a specific +pve version is required for a feature, use $version_guard
3450     # instead of min_version to allow machines to be run with the minimum
3451     # required version
3452     my $required_pve_version = 0;
3453     my $version_guard = sub {
3454         my ($major, $minor, $pve) = @_;
3455         return 0 if !min_version($machine_version, $major, $minor, $pve);
3456         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3457         return 1;
3458     };
3459
3460     if ($kvm) {
3461         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3462             if !defined kvm_version();
3463     }
3464
3465     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3466     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3467     my $use_old_bios_files = undef;
3468     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3469
3470     my $cpuunits = defined($conf->{cpuunits}) ?
3471             $conf->{cpuunits} : $defaults->{cpuunits};
3472
3473     push @$cmd, $kvm_binary;
3474
3475     push @$cmd, '-id', $vmid;
3476
3477     my $vmname = $conf->{name} || "vm$vmid";
3478
3479     push @$cmd, '-name', $vmname;
3480
3481     my $use_virtio = 0;
3482
3483     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3484     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3485     push @$cmd, '-mon', "chardev=qmp,mode=control";
3486
3487     if (min_version($machine_version, 2, 12)) {
3488         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3489         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3490     }
3491
3492     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3493
3494     push @$cmd, '-daemonize';
3495
3496     if ($conf->{smbios1}) {
3497         my $smbios_conf = parse_smbios1($conf->{smbios1});
3498         if ($smbios_conf->{base64}) {
3499             # Do not pass base64 flag to qemu
3500             delete $smbios_conf->{base64};
3501             my $smbios_string = "";
3502             foreach my $key (keys %$smbios_conf) {
3503                 my $value;
3504                 if ($key eq "uuid") {
3505                     $value = $smbios_conf->{uuid}
3506                 } else {
3507                     $value = decode_base64($smbios_conf->{$key});
3508                 }
3509                 # qemu accepts any binary data, only commas need escaping by double comma
3510                 $value =~ s/,/,,/g;
3511                 $smbios_string .= "," . $key . "=" . $value if $value;
3512             }
3513             push @$cmd, '-smbios', "type=1" . $smbios_string;
3514         } else {
3515             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3516         }
3517     }
3518
3519     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3520     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3521         die "uefi base image not found\n" if ! -f $ovmf_code;
3522
3523         my $path;
3524         my $format;
3525         if (my $efidisk = $conf->{efidisk0}) {
3526             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3527             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3528             $format = $d->{format};
3529             if ($storeid) {
3530                 $path = PVE::Storage::path($storecfg, $d->{file});
3531                 if (!defined($format)) {
3532                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3533                     $format = qemu_img_format($scfg, $volname);
3534                 }
3535             } else {
3536                 $path = $d->{file};
3537                 die "efidisk format must be specified\n"
3538                     if !defined($format);
3539             }
3540         } else {
3541             warn "no efidisk configured! Using temporary efivars disk.\n";
3542             $path = "/tmp/$vmid-ovmf.fd";
3543             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3544             $format = 'raw';
3545         }
3546
3547         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3548         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3549     }
3550
3551     # load q35 config
3552     if ($q35) {
3553         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3554         if (min_version($machine_version, 4, 0)) {
3555             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3556         } else {
3557             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3558         }
3559     }
3560
3561     if ($conf->{vmgenid}) {
3562         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3563     }
3564
3565     # add usb controllers
3566     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3567     push @$devices, @usbcontrollers if @usbcontrollers;
3568     my $vga = parse_vga($conf->{vga});
3569
3570     my $qxlnum = vga_conf_has_spice($conf->{vga});
3571     $vga->{type} = 'qxl' if $qxlnum;
3572
3573     if (!$vga->{type}) {
3574         if ($arch eq 'aarch64') {
3575             $vga->{type} = 'virtio';
3576         } elsif (min_version($machine_version, 2, 9)) {
3577             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3578         } else {
3579             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3580         }
3581     }
3582
3583     # enable absolute mouse coordinates (needed by vnc)
3584     my $tablet;
3585     if (defined($conf->{tablet})) {
3586         $tablet = $conf->{tablet};
3587     } else {
3588         $tablet = $defaults->{tablet};
3589         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3590         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3591     }
3592
3593     if ($tablet) {
3594         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3595         my $kbd = print_keyboarddevice_full($conf, $arch);
3596         push @$devices, '-device', $kbd if defined($kbd);
3597     }
3598
3599     my $kvm_off = 0;
3600     my $gpu_passthrough;
3601
3602     # host pci devices
3603     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3604         my $id = "hostpci$i";
3605         my $d = parse_hostpci($conf->{$id});
3606         next if !$d;
3607
3608         if (my $pcie = $d->{pcie}) {
3609             die "q35 machine model is not enabled" if !$q35;
3610             # win7 wants to have the pcie devices directly on the pcie bus
3611             # instead of in the root port
3612             if ($winversion == 7) {
3613                 $pciaddr = print_pcie_addr("${id}bus0");
3614             } else {
3615                 # add more root ports if needed, 4 are present by default
3616                 # by pve-q35 cfgs, rest added here on demand.
3617                 if ($i > 3) {
3618                     push @$devices, '-device', print_pcie_root_port($i);
3619                 }
3620                 $pciaddr = print_pcie_addr($id);
3621             }
3622         } else {
3623             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3624         }
3625
3626         my $xvga = '';
3627         if ($d->{'x-vga'}) {
3628             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3629             $kvm_off = 1;
3630             $vga->{type} = 'none' if !defined($conf->{vga});
3631             $gpu_passthrough = 1;
3632         }
3633
3634         my $pcidevices = $d->{pciid};
3635         my $multifunction = 1 if @$pcidevices > 1;
3636
3637         my $sysfspath;
3638         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3639             my $pci_id = $pcidevices->[0]->{id};
3640             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3641             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3642         } elsif ($d->{mdev}) {
3643             warn "ignoring mediated device '$id' with multifunction device\n";
3644         }
3645
3646         my $j=0;
3647         foreach my $pcidevice (@$pcidevices) {
3648             my $devicestr = "vfio-pci";
3649
3650             if ($sysfspath) {
3651                 $devicestr .= ",sysfsdev=$sysfspath";
3652             } else {
3653                 $devicestr .= ",host=$pcidevice->{id}";
3654             }
3655
3656             my $mf_addr = $multifunction ? ".$j" : '';
3657             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3658
3659             if ($j == 0) {
3660                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3661                 $devicestr .= "$xvga";
3662                 $devicestr .= ",multifunction=on" if $multifunction;
3663                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3664             }
3665
3666             push @$devices, '-device', $devicestr;
3667             $j++;
3668         }
3669     }
3670
3671     # usb devices
3672     my $usb_dev_features = {};
3673     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3674
3675     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3676     push @$devices, @usbdevices if @usbdevices;
3677     # serial devices
3678     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3679         if (my $path = $conf->{"serial$i"}) {
3680             if ($path eq 'socket') {
3681                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3682                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3683                 # On aarch64, serial0 is the UART device. Qemu only allows
3684                 # connecting UART devices via the '-serial' command line, as
3685                 # the device has a fixed slot on the hardware...
3686                 if ($arch eq 'aarch64' && $i == 0) {
3687                     push @$devices, '-serial', "chardev:serial$i";
3688                 } else {
3689                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3690                 }
3691             } else {
3692                 die "no such serial device\n" if ! -c $path;
3693                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3694                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3695             }
3696         }
3697     }
3698
3699     # parallel devices
3700     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3701         if (my $path = $conf->{"parallel$i"}) {
3702             die "no such parallel device\n" if ! -c $path;
3703             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3704             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3705             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3706         }
3707     }
3708
3709     if (my $audio = conf_has_audio($conf)) {
3710
3711         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3712
3713         my $id = $audio->{dev_id};
3714         if ($audio->{dev} eq 'AC97') {
3715             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3716         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3717             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3718             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3719             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3720         } else {
3721             die "unkown audio device '$audio->{dev}', implement me!";
3722         }
3723
3724         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3725     }
3726
3727     my $sockets = 1;
3728     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3729     $sockets = $conf->{sockets} if  $conf->{sockets};
3730
3731     my $cores = $conf->{cores} || 1;
3732
3733     my $maxcpus = $sockets * $cores;
3734
3735     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3736
3737     my $allowed_vcpus = $cpuinfo->{cpus};
3738
3739     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3740         if ($allowed_vcpus < $maxcpus);
3741
3742     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3743
3744         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3745         for (my $i = 2; $i <= $vcpus; $i++)  {
3746             my $cpustr = print_cpu_device($conf,$i);
3747             push @$cmd, '-device', $cpustr;
3748         }
3749
3750     } else {
3751
3752         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753     }
3754     push @$cmd, '-nodefaults';
3755
3756     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3757
3758     my $bootindex_hash = {};
3759     my $i = 1;
3760     foreach my $o (split(//, $bootorder)) {
3761         $bootindex_hash->{$o} = $i*100;
3762         $i++;
3763     }
3764
3765     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3766
3767     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3768
3769     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3770
3771     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3772         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3773         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3774         push @$cmd,  '-vnc', "unix:$socket,password";
3775     } else {
3776         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3777         push @$cmd, '-nographic';
3778     }
3779
3780     # time drift fix
3781     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3782
3783     my $useLocaltime = $conf->{localtime};
3784
3785     if ($winversion >= 5) { # windows
3786         $useLocaltime = 1 if !defined($conf->{localtime});
3787
3788         # use time drift fix when acpi is enabled
3789         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3790             $tdf = 1 if !defined($conf->{tdf});
3791         }
3792     }
3793
3794     if ($winversion >= 6) {
3795         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3796         push @$cmd, '-no-hpet';
3797     }
3798
3799     push @$rtcFlags, 'driftfix=slew' if $tdf;
3800
3801     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3802         push @$rtcFlags, "base=$conf->{startdate}";
3803     } elsif ($useLocaltime) {
3804         push @$rtcFlags, 'base=localtime';
3805     }
3806
3807     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3808
3809     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3810
3811     push @$cmd, '-S' if $conf->{freeze};
3812
3813     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3814
3815     my $guest_agent = parse_guest_agent($conf);
3816
3817     if ($guest_agent->{enabled}) {
3818         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3819         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3820
3821         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3822             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3823             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3824             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3825         } elsif ($guest_agent->{type} eq 'isa') {
3826             push @$devices, '-device', "isa-serial,chardev=qga0";
3827         }
3828     }
3829
3830     my $spice_port;
3831
3832     if ($qxlnum) {
3833         if ($qxlnum > 1) {
3834             if ($winversion){
3835                 for(my $i = 1; $i < $qxlnum; $i++){
3836                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3837                 }
3838             } else {
3839                 # assume other OS works like Linux
3840                 my ($ram, $vram) = ("134217728", "67108864");
3841                 if ($vga->{memory}) {
3842                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3843                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3844                 }
3845                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3846                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3847             }
3848         }
3849
3850         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3851
3852         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3853         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3854         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3855
3856         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3857         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3858         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3859
3860         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3861         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3862
3863         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3864         if ($spice_enhancement->{foldersharing}) {
3865             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3866             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3867         }
3868
3869         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3870         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3871         push @$devices, '-spice', "$spice_opts";
3872     }
3873
3874     # enable balloon by default, unless explicitly disabled
3875     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3876         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3877         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3878     }
3879
3880     if ($conf->{watchdog}) {
3881         my $wdopts = parse_watchdog($conf->{watchdog});
3882         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3883         my $watchdog = $wdopts->{model} || 'i6300esb';
3884         push @$devices, '-device', "$watchdog$pciaddr";
3885         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3886     }
3887
3888     my $vollist = [];
3889     my $scsicontroller = {};
3890     my $ahcicontroller = {};
3891     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3892
3893     # Add iscsi initiator name if available
3894     if (my $initiator = get_initiator_name()) {
3895         push @$devices, '-iscsi', "initiator-name=$initiator";
3896     }
3897
3898     foreach_drive($conf, sub {
3899         my ($ds, $drive) = @_;
3900
3901         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3902             push @$vollist, $drive->{file};
3903         }
3904
3905         # ignore efidisk here, already added in bios/fw handling code above
3906         return if $drive->{interface} eq 'efidisk';
3907
3908         $use_virtio = 1 if $ds =~ m/^virtio/;
3909
3910         if (drive_is_cdrom ($drive)) {
3911             if ($bootindex_hash->{d}) {
3912                 $drive->{bootindex} = $bootindex_hash->{d};
3913                 $bootindex_hash->{d} += 1;
3914             }
3915         } else {
3916             if ($bootindex_hash->{c}) {
3917                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3918                 $bootindex_hash->{c} += 1;
3919             }
3920         }
3921
3922         if($drive->{interface} eq 'virtio'){
3923            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3924         }
3925
3926         if ($drive->{interface} eq 'scsi') {
3927
3928             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3929
3930             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3931                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3932
3933             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3934             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3935
3936             my $iothread = '';
3937             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3938                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3939                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3940             } elsif ($drive->{iothread}) {
3941                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3942             }
3943
3944             my $queues = '';
3945             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3946                 $queues = ",num_queues=$drive->{queues}";
3947             }
3948
3949             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3950             $scsicontroller->{$controller}=1;
3951         }
3952
3953         if ($drive->{interface} eq 'sata') {
3954            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3955            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3956            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3957            $ahcicontroller->{$controller}=1;
3958         }
3959
3960         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3961         push @$devices, '-drive',$drive_cmd;
3962         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3963     });
3964
3965     for (my $i = 0; $i < $MAX_NETS; $i++) {
3966          next if !$conf->{"net$i"};
3967          my $d = parse_net($conf->{"net$i"});
3968          next if !$d;
3969
3970          $use_virtio = 1 if $d->{model} eq 'virtio';
3971
3972          if ($bootindex_hash->{n}) {
3973             $d->{bootindex} = $bootindex_hash->{n};
3974             $bootindex_hash->{n} += 1;
3975          }
3976
3977          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3978          push @$devices, '-netdev', $netdevfull;
3979
3980          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3981          push @$devices, '-device', $netdevicefull;
3982     }
3983
3984     if ($conf->{ivshmem}) {
3985         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3986
3987         my $bus;
3988         if ($q35) {
3989             $bus = print_pcie_addr("ivshmem");
3990         } else {
3991             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3992         }
3993
3994         my $ivshmem_name = $ivshmem->{name} // $vmid;
3995         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3996
3997         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3998         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3999     }
4000
4001     # pci.4 is nested in pci.1
4002     $bridges->{1} = 1 if $bridges->{4};
4003
4004     if (!$q35) {
4005         # add pci bridges
4006         if (min_version($machine_version, 2, 3)) {
4007            $bridges->{1} = 1;
4008            $bridges->{2} = 1;
4009         }
4010
4011         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4012
4013     }
4014
4015     for my $k (sort {$b cmp $a} keys %$bridges) {
4016         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4017         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4018         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4019         if ($q35) {
4020             # add after -readconfig pve-q35.cfg
4021             splice @$devices, 2, 0, '-device', $devstr;
4022         } else {
4023             unshift @$devices, '-device', $devstr if $k > 0;
4024         }
4025     }
4026
4027     if (!$kvm) {
4028         push @$machineFlags, 'accel=tcg';
4029     }
4030
4031     my $machine_type_min = $machine_type;
4032     if ($add_pve_version) {
4033         $machine_type_min =~ s/\+pve\d+$//;
4034         $machine_type_min .= "+pve$required_pve_version";
4035     }
4036     push @$machineFlags, "type=${machine_type_min}";
4037
4038     push @$cmd, @$devices;
4039     push @$cmd, '-rtc', join(',', @$rtcFlags)
4040         if scalar(@$rtcFlags);
4041     push @$cmd, '-machine', join(',', @$machineFlags)
4042         if scalar(@$machineFlags);
4043     push @$cmd, '-global', join(',', @$globalFlags)
4044         if scalar(@$globalFlags);
4045
4046     if (my $vmstate = $conf->{vmstate}) {
4047         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4048         push @$vollist, $vmstate;
4049         push @$cmd, '-loadstate', $statepath;
4050         print "activating and using '$vmstate' as vmstate\n";
4051     }
4052
4053     # add custom args
4054     if ($conf->{args}) {
4055         my $aa = PVE::Tools::split_args($conf->{args});
4056         push @$cmd, @$aa;
4057     }
4058
4059     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4060 }
4061
4062 sub spice_port {
4063     my ($vmid) = @_;
4064
4065     my $res = mon_cmd($vmid, 'query-spice');
4066
4067     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4068 }
4069
4070 sub vm_devices_list {
4071     my ($vmid) = @_;
4072
4073     my $res = mon_cmd($vmid, 'query-pci');
4074     my $devices_to_check = [];
4075     my $devices = {};
4076     foreach my $pcibus (@$res) {
4077         push @$devices_to_check, @{$pcibus->{devices}},
4078     }
4079
4080     while (@$devices_to_check) {
4081         my $to_check = [];
4082         for my $d (@$devices_to_check) {
4083             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4084             next if !$d->{'pci_bridge'};
4085
4086             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4087             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4088         }
4089         $devices_to_check = $to_check;
4090     }
4091
4092     my $resblock = mon_cmd($vmid, 'query-block');
4093     foreach my $block (@$resblock) {
4094         if($block->{device} =~ m/^drive-(\S+)/){
4095                 $devices->{$1} = 1;
4096         }
4097     }
4098
4099     my $resmice = mon_cmd($vmid, 'query-mice');
4100     foreach my $mice (@$resmice) {
4101         if ($mice->{name} eq 'QEMU HID Tablet') {
4102             $devices->{tablet} = 1;
4103             last;
4104         }
4105     }
4106
4107     # for usb devices there is no query-usb
4108     # but we can iterate over the entries in
4109     # qom-list path=/machine/peripheral
4110     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4111     foreach my $per (@$resperipheral) {
4112         if ($per->{name} =~ m/^usb\d+$/) {
4113             $devices->{$per->{name}} = 1;
4114         }
4115     }
4116
4117     return $devices;
4118 }
4119
4120 sub vm_deviceplug {
4121     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4122
4123     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4124
4125     my $devices_list = vm_devices_list($vmid);
4126     return 1 if defined($devices_list->{$deviceid});
4127
4128     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4129
4130     if ($deviceid eq 'tablet') {
4131
4132         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4133
4134     } elsif ($deviceid eq 'keyboard') {
4135
4136         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4137
4138     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4139
4140         die "usb hotplug currently not reliable\n";
4141         # since we can't reliably hot unplug all added usb devices
4142         # and usb passthrough disables live migration
4143         # we disable usb hotplugging for now
4144         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4145
4146     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4147
4148         qemu_iothread_add($vmid, $deviceid, $device);
4149
4150         qemu_driveadd($storecfg, $vmid, $device);
4151         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4152
4153         qemu_deviceadd($vmid, $devicefull);
4154         eval { qemu_deviceaddverify($vmid, $deviceid); };
4155         if (my $err = $@) {
4156             eval { qemu_drivedel($vmid, $deviceid); };
4157             warn $@ if $@;
4158             die $err;
4159         }
4160
4161     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4162
4163
4164         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4165         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4166         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4167
4168         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4169
4170         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4171             qemu_iothread_add($vmid, $deviceid, $device);
4172             $devicefull .= ",iothread=iothread-$deviceid";
4173         }
4174
4175         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4176             $devicefull .= ",num_queues=$device->{queues}";
4177         }
4178
4179         qemu_deviceadd($vmid, $devicefull);
4180         qemu_deviceaddverify($vmid, $deviceid);
4181
4182     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4183
4184         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4185         qemu_driveadd($storecfg, $vmid, $device);
4186
4187         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4188         eval { qemu_deviceadd($vmid, $devicefull); };
4189         if (my $err = $@) {
4190             eval { qemu_drivedel($vmid, $deviceid); };
4191             warn $@ if $@;
4192             die $err;
4193         }
4194
4195     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4196
4197         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4198
4199         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4200         my $use_old_bios_files = undef;
4201         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4202
4203         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4204         qemu_deviceadd($vmid, $netdevicefull);
4205         eval {
4206             qemu_deviceaddverify($vmid, $deviceid);
4207             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4208         };
4209         if (my $err = $@) {
4210             eval { qemu_netdevdel($vmid, $deviceid); };
4211             warn $@ if $@;
4212             die $err;
4213         }
4214
4215     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4216
4217         my $bridgeid = $2;
4218         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4219         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4220
4221         qemu_deviceadd($vmid, $devicefull);
4222         qemu_deviceaddverify($vmid, $deviceid);
4223
4224     } else {
4225         die "can't hotplug device '$deviceid'\n";
4226     }
4227
4228     return 1;
4229 }
4230
4231 # fixme: this should raise exceptions on error!
4232 sub vm_deviceunplug {
4233     my ($vmid, $conf, $deviceid) = @_;
4234
4235     my $devices_list = vm_devices_list($vmid);
4236     return 1 if !defined($devices_list->{$deviceid});
4237
4238     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4239
4240     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4241
4242         qemu_devicedel($vmid, $deviceid);
4243
4244     } elsif ($deviceid =~ m/^usb\d+$/) {
4245
4246         die "usb hotplug currently not reliable\n";
4247         # when unplugging usb devices this way,
4248         # there may be remaining usb controllers/hubs
4249         # so we disable it for now
4250         qemu_devicedel($vmid, $deviceid);
4251         qemu_devicedelverify($vmid, $deviceid);
4252
4253     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4254
4255         qemu_devicedel($vmid, $deviceid);
4256         qemu_devicedelverify($vmid, $deviceid);
4257         qemu_drivedel($vmid, $deviceid);
4258         qemu_iothread_del($conf, $vmid, $deviceid);
4259
4260     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4261
4262         qemu_devicedel($vmid, $deviceid);
4263         qemu_devicedelverify($vmid, $deviceid);
4264         qemu_iothread_del($conf, $vmid, $deviceid);
4265
4266     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4267
4268         qemu_devicedel($vmid, $deviceid);
4269         qemu_drivedel($vmid, $deviceid);
4270         qemu_deletescsihw($conf, $vmid, $deviceid);
4271
4272     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4273
4274         qemu_devicedel($vmid, $deviceid);
4275         qemu_devicedelverify($vmid, $deviceid);
4276         qemu_netdevdel($vmid, $deviceid);
4277
4278     } else {
4279         die "can't unplug device '$deviceid'\n";
4280     }
4281
4282     return 1;
4283 }
4284
4285 sub qemu_deviceadd {
4286     my ($vmid, $devicefull) = @_;
4287
4288     $devicefull = "driver=".$devicefull;
4289     my %options =  split(/[=,]/, $devicefull);
4290
4291     mon_cmd($vmid, "device_add" , %options);
4292 }
4293
4294 sub qemu_devicedel {
4295     my ($vmid, $deviceid) = @_;
4296
4297     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4298 }
4299
4300 sub qemu_iothread_add {
4301     my($vmid, $deviceid, $device) = @_;
4302
4303     if ($device->{iothread}) {
4304         my $iothreads = vm_iothreads_list($vmid);
4305         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4306     }
4307 }
4308
4309 sub qemu_iothread_del {
4310     my($conf, $vmid, $deviceid) = @_;
4311
4312     my $confid = $deviceid;
4313     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4314         $confid = 'scsi' . $1;
4315     }
4316     my $device = parse_drive($confid, $conf->{$confid});
4317     if ($device->{iothread}) {
4318         my $iothreads = vm_iothreads_list($vmid);
4319         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4320     }
4321 }
4322
4323 sub qemu_objectadd {
4324     my($vmid, $objectid, $qomtype) = @_;
4325
4326     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4327
4328     return 1;
4329 }
4330
4331 sub qemu_objectdel {
4332     my($vmid, $objectid) = @_;
4333
4334     mon_cmd($vmid, "object-del", id => $objectid);
4335
4336     return 1;
4337 }
4338
4339 sub qemu_driveadd {
4340     my ($storecfg, $vmid, $device) = @_;
4341
4342     my $drive = print_drive_full($storecfg, $vmid, $device);
4343     $drive =~ s/\\/\\\\/g;
4344     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4345
4346     # If the command succeeds qemu prints: "OK"
4347     return 1 if $ret =~ m/OK/s;
4348
4349     die "adding drive failed: $ret\n";
4350 }
4351
4352 sub qemu_drivedel {
4353     my($vmid, $deviceid) = @_;
4354
4355     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4356     $ret =~ s/^\s+//;
4357
4358     return 1 if $ret eq "";
4359
4360     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4361     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4362
4363     die "deleting drive $deviceid failed : $ret\n";
4364 }
4365
4366 sub qemu_deviceaddverify {
4367     my ($vmid, $deviceid) = @_;
4368
4369     for (my $i = 0; $i <= 5; $i++) {
4370          my $devices_list = vm_devices_list($vmid);
4371          return 1 if defined($devices_list->{$deviceid});
4372          sleep 1;
4373     }
4374
4375     die "error on hotplug device '$deviceid'\n";
4376 }
4377
4378
4379 sub qemu_devicedelverify {
4380     my ($vmid, $deviceid) = @_;
4381
4382     # need to verify that the device is correctly removed as device_del
4383     # is async and empty return is not reliable
4384
4385     for (my $i = 0; $i <= 5; $i++) {
4386          my $devices_list = vm_devices_list($vmid);
4387          return 1 if !defined($devices_list->{$deviceid});
4388          sleep 1;
4389     }
4390
4391     die "error on hot-unplugging device '$deviceid'\n";
4392 }
4393
4394 sub qemu_findorcreatescsihw {
4395     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4396
4397     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4398
4399     my $scsihwid="$controller_prefix$controller";
4400     my $devices_list = vm_devices_list($vmid);
4401
4402     if(!defined($devices_list->{$scsihwid})) {
4403         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4404     }
4405
4406     return 1;
4407 }
4408
4409 sub qemu_deletescsihw {
4410     my ($conf, $vmid, $opt) = @_;
4411
4412     my $device = parse_drive($opt, $conf->{$opt});
4413
4414     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4415         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4416         return 1;
4417     }
4418
4419     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4420
4421     my $devices_list = vm_devices_list($vmid);
4422     foreach my $opt (keys %{$devices_list}) {
4423         if (PVE::QemuServer::is_valid_drivename($opt)) {
4424             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4425             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4426                 return 1;
4427             }
4428         }
4429     }
4430
4431     my $scsihwid="scsihw$controller";
4432
4433     vm_deviceunplug($vmid, $conf, $scsihwid);
4434
4435     return 1;
4436 }
4437
4438 sub qemu_add_pci_bridge {
4439     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4440
4441     my $bridges = {};
4442
4443     my $bridgeid;
4444
4445     print_pci_addr($device, $bridges, $arch, $machine_type);
4446
4447     while (my ($k, $v) = each %$bridges) {
4448         $bridgeid = $k;
4449     }
4450     return 1 if !defined($bridgeid) || $bridgeid < 1;
4451
4452     my $bridge = "pci.$bridgeid";
4453     my $devices_list = vm_devices_list($vmid);
4454
4455     if (!defined($devices_list->{$bridge})) {
4456         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4457     }
4458
4459     return 1;
4460 }
4461
4462 sub qemu_set_link_status {
4463     my ($vmid, $device, $up) = @_;
4464
4465     mon_cmd($vmid, "set_link", name => $device,
4466                up => $up ? JSON::true : JSON::false);
4467 }
4468
4469 sub qemu_netdevadd {
4470     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4471
4472     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4473     my %options =  split(/[=,]/, $netdev);
4474
4475     mon_cmd($vmid, "netdev_add",  %options);
4476     return 1;
4477 }
4478
4479 sub qemu_netdevdel {
4480     my ($vmid, $deviceid) = @_;
4481
4482     mon_cmd($vmid, "netdev_del", id => $deviceid);
4483 }
4484
4485 sub qemu_usb_hotplug {
4486     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4487
4488     return if !$device;
4489
4490     # remove the old one first
4491     vm_deviceunplug($vmid, $conf, $deviceid);
4492
4493     # check if xhci controller is necessary and available
4494     if ($device->{usb3}) {
4495
4496         my $devicelist = vm_devices_list($vmid);
4497
4498         if (!$devicelist->{xhci}) {
4499             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4500             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4501         }
4502     }
4503     my $d = parse_usb_device($device->{host});
4504     $d->{usb3} = $device->{usb3};
4505
4506     # add the new one
4507     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4508 }
4509
4510 sub qemu_cpu_hotplug {
4511     my ($vmid, $conf, $vcpus) = @_;
4512
4513     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4514
4515     my $sockets = 1;
4516     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4517     $sockets = $conf->{sockets} if  $conf->{sockets};
4518     my $cores = $conf->{cores} || 1;
4519     my $maxcpus = $sockets * $cores;
4520
4521     $vcpus = $maxcpus if !$vcpus;
4522
4523     die "you can't add more vcpus than maxcpus\n"
4524         if $vcpus > $maxcpus;
4525
4526     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4527
4528     if ($vcpus < $currentvcpus) {
4529
4530         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4531
4532             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4533                 qemu_devicedel($vmid, "cpu$i");
4534                 my $retry = 0;
4535                 my $currentrunningvcpus = undef;
4536                 while (1) {
4537                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4538                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4539                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4540                     $retry++;
4541                     sleep 1;
4542                 }
4543                 #update conf after each succesfull cpu unplug
4544                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4545                 PVE::QemuConfig->write_config($vmid, $conf);
4546             }
4547         } else {
4548             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4549         }
4550
4551         return;
4552     }
4553
4554     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4555     die "vcpus in running vm does not match its configuration\n"
4556         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4557
4558     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4559
4560         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4561             my $cpustr = print_cpu_device($conf, $i);
4562             qemu_deviceadd($vmid, $cpustr);
4563
4564             my $retry = 0;
4565             my $currentrunningvcpus = undef;
4566             while (1) {
4567                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4568                 last if scalar(@{$currentrunningvcpus}) == $i;
4569                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4570                 sleep 1;
4571                 $retry++;
4572             }
4573             #update conf after each succesfull cpu hotplug
4574             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4575             PVE::QemuConfig->write_config($vmid, $conf);
4576         }
4577     } else {
4578
4579         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4580             mon_cmd($vmid, "cpu-add", id => int($i));
4581         }
4582     }
4583 }
4584
4585 sub qemu_block_set_io_throttle {
4586     my ($vmid, $deviceid,
4587         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4588         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4589         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4590         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4591
4592     return if !check_running($vmid) ;
4593
4594     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4595         bps => int($bps),
4596         bps_rd => int($bps_rd),
4597         bps_wr => int($bps_wr),
4598         iops => int($iops),
4599         iops_rd => int($iops_rd),
4600         iops_wr => int($iops_wr),
4601         bps_max => int($bps_max),
4602         bps_rd_max => int($bps_rd_max),
4603         bps_wr_max => int($bps_wr_max),
4604         iops_max => int($iops_max),
4605         iops_rd_max => int($iops_rd_max),
4606         iops_wr_max => int($iops_wr_max),
4607         bps_max_length => int($bps_max_length),
4608         bps_rd_max_length => int($bps_rd_max_length),
4609         bps_wr_max_length => int($bps_wr_max_length),
4610         iops_max_length => int($iops_max_length),
4611         iops_rd_max_length => int($iops_rd_max_length),
4612         iops_wr_max_length => int($iops_wr_max_length),
4613     );
4614
4615 }
4616
4617 # old code, only used to shutdown old VM after update
4618 sub __read_avail {
4619     my ($fh, $timeout) = @_;
4620
4621     my $sel = new IO::Select;
4622     $sel->add($fh);
4623
4624     my $res = '';
4625     my $buf;
4626
4627     my @ready;
4628     while (scalar (@ready = $sel->can_read($timeout))) {
4629         my $count;
4630         if ($count = $fh->sysread($buf, 8192)) {
4631             if ($buf =~ /^(.*)\(qemu\) $/s) {
4632                 $res .= $1;
4633                 last;
4634             } else {
4635                 $res .= $buf;
4636             }
4637         } else {
4638             if (!defined($count)) {
4639                 die "$!\n";
4640             }
4641             last;
4642         }
4643     }
4644
4645     die "monitor read timeout\n" if !scalar(@ready);
4646
4647     return $res;
4648 }
4649
4650 sub qemu_block_resize {
4651     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4652
4653     my $running = check_running($vmid);
4654
4655     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4656
4657     return if !$running;
4658
4659     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4660
4661 }
4662
4663 sub qemu_volume_snapshot {
4664     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4665
4666     my $running = check_running($vmid);
4667
4668     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4669         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4670     } else {
4671         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4672     }
4673 }
4674
4675 sub qemu_volume_snapshot_delete {
4676     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4677
4678     my $running = check_running($vmid);
4679
4680     if($running) {
4681
4682         $running = undef;
4683         my $conf = PVE::QemuConfig->load_config($vmid);
4684         foreach_drive($conf, sub {
4685             my ($ds, $drive) = @_;
4686             $running = 1 if $drive->{file} eq $volid;
4687         });
4688     }
4689
4690     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4691         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4692     } else {
4693         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4694     }
4695 }
4696
4697 sub set_migration_caps {
4698     my ($vmid) = @_;
4699
4700     my $cap_ref = [];
4701
4702     my $enabled_cap = {
4703         "auto-converge" => 1,
4704         "xbzrle" => 1,
4705         "x-rdma-pin-all" => 0,
4706         "zero-blocks" => 0,
4707         "compress" => 0
4708     };
4709
4710     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4711
4712     for my $supported_capability (@$supported_capabilities) {
4713         push @$cap_ref, {
4714             capability => $supported_capability->{capability},
4715             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4716         };
4717     }
4718
4719     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4720 }
4721
4722 my $fast_plug_option = {
4723     'lock' => 1,
4724     'name' => 1,
4725     'onboot' => 1,
4726     'shares' => 1,
4727     'startup' => 1,
4728     'description' => 1,
4729     'protection' => 1,
4730     'vmstatestorage' => 1,
4731     'hookscript' => 1,
4732     'tags' => 1,
4733 };
4734
4735 # hotplug changes in [PENDING]
4736 # $selection hash can be used to only apply specified options, for
4737 # example: { cores => 1 } (only apply changed 'cores')
4738 # $errors ref is used to return error messages
4739 sub vmconfig_hotplug_pending {
4740     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4741
4742     my $defaults = load_defaults();
4743     my $arch = get_vm_arch($conf);
4744     my $machine_type = get_vm_machine($conf, undef, $arch);
4745
4746     # commit values which do not have any impact on running VM first
4747     # Note: those option cannot raise errors, we we do not care about
4748     # $selection and always apply them.
4749
4750     my $add_error = sub {
4751         my ($opt, $msg) = @_;
4752         $errors->{$opt} = "hotplug problem - $msg";
4753     };
4754
4755     my $changes = 0;
4756     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4757         if ($fast_plug_option->{$opt}) {
4758             $conf->{$opt} = $conf->{pending}->{$opt};
4759             delete $conf->{pending}->{$opt};
4760             $changes = 1;
4761         }
4762     }
4763
4764     if ($changes) {
4765         PVE::QemuConfig->write_config($vmid, $conf);
4766     }
4767
4768     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4769
4770     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4771     foreach my $opt (sort keys %$pending_delete_hash) {
4772         next if $selection && !$selection->{$opt};
4773         my $force = $pending_delete_hash->{$opt}->{force};
4774         eval {
4775             if ($opt eq 'hotplug') {
4776                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4777             } elsif ($opt eq 'tablet') {
4778                 die "skip\n" if !$hotplug_features->{usb};
4779                 if ($defaults->{tablet}) {
4780                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4781                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4782                         if $arch eq 'aarch64';
4783                 } else {
4784                     vm_deviceunplug($vmid, $conf, 'tablet');
4785                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4786                 }
4787             } elsif ($opt =~ m/^usb\d+/) {
4788                 die "skip\n";
4789                 # since we cannot reliably hot unplug usb devices
4790                 # we are disabling it
4791                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4792                 vm_deviceunplug($vmid, $conf, $opt);
4793             } elsif ($opt eq 'vcpus') {
4794                 die "skip\n" if !$hotplug_features->{cpu};
4795                 qemu_cpu_hotplug($vmid, $conf, undef);
4796             } elsif ($opt eq 'balloon') {
4797                 # enable balloon device is not hotpluggable
4798                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4799                 # here we reset the ballooning value to memory
4800                 my $balloon = $conf->{memory} || $defaults->{memory};
4801                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4802             } elsif ($fast_plug_option->{$opt}) {
4803                 # do nothing
4804             } elsif ($opt =~ m/^net(\d+)$/) {
4805                 die "skip\n" if !$hotplug_features->{network};
4806                 vm_deviceunplug($vmid, $conf, $opt);
4807             } elsif (is_valid_drivename($opt)) {
4808                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4809                 vm_deviceunplug($vmid, $conf, $opt);
4810                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4811             } elsif ($opt =~ m/^memory$/) {
4812                 die "skip\n" if !$hotplug_features->{memory};
4813                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4814             } elsif ($opt eq 'cpuunits') {
4815                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4816             } elsif ($opt eq 'cpulimit') {
4817                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4818             } else {
4819                 die "skip\n";
4820             }
4821         };
4822         if (my $err = $@) {
4823             &$add_error($opt, $err) if $err ne "skip\n";
4824         } else {
4825             delete $conf->{$opt};
4826             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4827         }
4828     }
4829
4830     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4831     $apply_pending_cloudinit = sub {
4832         return if $apply_pending_cloudinit_done; # once is enough
4833         $apply_pending_cloudinit_done = 1; # once is enough
4834
4835         my ($key, $value) = @_;
4836
4837         my @cloudinit_opts = keys %$confdesc_cloudinit;
4838         foreach my $opt (keys %{$conf->{pending}}) {
4839             next if !grep { $_ eq $opt } @cloudinit_opts;
4840             $conf->{$opt} = delete $conf->{pending}->{$opt};
4841         }
4842
4843         my $new_conf = { %$conf };
4844         $new_conf->{$key} = $value;
4845         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4846     };
4847
4848     foreach my $opt (keys %{$conf->{pending}}) {
4849         next if $selection && !$selection->{$opt};
4850         my $value = $conf->{pending}->{$opt};
4851         eval {
4852             if ($opt eq 'hotplug') {
4853                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4854             } elsif ($opt eq 'tablet') {
4855                 die "skip\n" if !$hotplug_features->{usb};
4856                 if ($value == 1) {
4857                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4858                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4859                         if $arch eq 'aarch64';
4860                 } elsif ($value == 0) {
4861                     vm_deviceunplug($vmid, $conf, 'tablet');
4862                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4863                 }
4864             } elsif ($opt =~ m/^usb\d+$/) {
4865                 die "skip\n";
4866                 # since we cannot reliably hot unplug usb devices
4867                 # we are disabling it
4868                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4869                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4870                 die "skip\n" if !$d;
4871                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4872             } elsif ($opt eq 'vcpus') {
4873                 die "skip\n" if !$hotplug_features->{cpu};
4874                 qemu_cpu_hotplug($vmid, $conf, $value);
4875             } elsif ($opt eq 'balloon') {
4876                 # enable/disable balloning device is not hotpluggable
4877                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4878                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4879                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4880
4881                 # allow manual ballooning if shares is set to zero
4882                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4883                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4884                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4885                 }
4886             } elsif ($opt =~ m/^net(\d+)$/) {
4887                 # some changes can be done without hotplug
4888                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4889                                     $vmid, $opt, $value, $arch, $machine_type);
4890             } elsif (is_valid_drivename($opt)) {
4891                 die "skip\n" if $opt eq 'efidisk0';
4892                 # some changes can be done without hotplug
4893                 my $drive = parse_drive($opt, $value);
4894                 if (drive_is_cloudinit($drive)) {
4895                     &$apply_pending_cloudinit($opt, $value);
4896                 }
4897                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4898                                      $vmid, $opt, $value, $arch, $machine_type);
4899             } elsif ($opt =~ m/^memory$/) { #dimms
4900                 die "skip\n" if !$hotplug_features->{memory};
4901                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4902             } elsif ($opt eq 'cpuunits') {
4903                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4904             } elsif ($opt eq 'cpulimit') {
4905                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4906                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4907             } else {
4908                 die "skip\n";  # skip non-hot-pluggable options
4909             }
4910         };
4911         if (my $err = $@) {
4912             &$add_error($opt, $err) if $err ne "skip\n";
4913         } else {
4914             $conf->{$opt} = $value;
4915             delete $conf->{pending}->{$opt};
4916         }
4917     }
4918
4919     PVE::QemuConfig->write_config($vmid, $conf);
4920 }
4921
4922 sub try_deallocate_drive {
4923     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4924
4925     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4926         my $volid = $drive->{file};
4927         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4928             my $sid = PVE::Storage::parse_volume_id($volid);
4929             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4930
4931             # check if the disk is really unused
4932             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4933                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4934             PVE::Storage::vdisk_free($storecfg, $volid);
4935             return 1;
4936         } else {
4937             # If vm is not owner of this disk remove from config
4938             return 1;
4939         }
4940     }
4941
4942     return undef;
4943 }
4944
4945 sub vmconfig_delete_or_detach_drive {
4946     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4947
4948     my $drive = parse_drive($opt, $conf->{$opt});
4949
4950     my $rpcenv = PVE::RPCEnvironment::get();
4951     my $authuser = $rpcenv->get_user();
4952
4953     if ($force) {
4954         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4955         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4956     } else {
4957         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4958     }
4959 }
4960
4961
4962
4963 sub vmconfig_apply_pending {
4964     my ($vmid, $conf, $storecfg, $errors) = @_;
4965
4966     my $add_apply_error = sub {
4967         my ($opt, $msg) = @_;
4968         my $err_msg = "unable to apply pending change $opt : $msg";
4969         $errors->{$opt} = $err_msg;
4970         warn $err_msg;
4971     };
4972
4973     # cold plug
4974
4975     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4976     foreach my $opt (sort keys %$pending_delete_hash) {
4977         my $force = $pending_delete_hash->{$opt}->{force};
4978         eval {
4979             if ($opt =~ m/^unused/) {
4980                 die "internal error";
4981             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4982                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4983             }
4984         };
4985         if (my $err = $@) {
4986             $add_apply_error->($opt, $err);
4987         } else {
4988             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4989             delete $conf->{$opt};
4990         }
4991     }
4992
4993     PVE::QemuConfig->cleanup_pending($conf);
4994
4995     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4996         next if $opt eq 'delete'; # just to be sure
4997         eval {
4998             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4999                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5000             }
5001         };
5002         if (my $err = $@) {
5003             $add_apply_error->($opt, $err);
5004         } else {
5005             $conf->{$opt} = delete $conf->{pending}->{$opt};
5006         }
5007     }
5008
5009     # write all changes at once to avoid unnecessary i/o
5010     PVE::QemuConfig->write_config($vmid, $conf);
5011 }
5012
5013 my $safe_num_ne = sub {
5014     my ($a, $b) = @_;
5015
5016     return 0 if !defined($a) && !defined($b);
5017     return 1 if !defined($a);
5018     return 1 if !defined($b);
5019
5020     return $a != $b;
5021 };
5022
5023 my $safe_string_ne = sub {
5024     my ($a, $b) = @_;
5025
5026     return 0 if !defined($a) && !defined($b);
5027     return 1 if !defined($a);
5028     return 1 if !defined($b);
5029
5030     return $a ne $b;
5031 };
5032
5033 sub vmconfig_update_net {
5034     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5035
5036     my $newnet = parse_net($value);
5037
5038     if ($conf->{$opt}) {
5039         my $oldnet = parse_net($conf->{$opt});
5040
5041         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5042             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5043             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5044             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5045
5046             # for non online change, we try to hot-unplug
5047             die "skip\n" if !$hotplug;
5048             vm_deviceunplug($vmid, $conf, $opt);
5049         } else {
5050
5051             die "internal error" if $opt !~ m/net(\d+)/;
5052             my $iface = "tap${vmid}i$1";
5053
5054             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5055                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5056                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5057                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5058                 PVE::Network::tap_unplug($iface);
5059                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5060             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5061                 # Rate can be applied on its own but any change above needs to
5062                 # include the rate in tap_plug since OVS resets everything.
5063                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5064             }
5065
5066             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5067                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5068             }
5069
5070             return 1;
5071         }
5072     }
5073
5074     if ($hotplug) {
5075         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5076     } else {
5077         die "skip\n";
5078     }
5079 }
5080
5081 sub vmconfig_update_disk {
5082     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5083
5084     my $drive = parse_drive($opt, $value);
5085
5086     if ($conf->{$opt}) {
5087
5088         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5089
5090             my $media = $drive->{media} || 'disk';
5091             my $oldmedia = $old_drive->{media} || 'disk';
5092             die "unable to change media type\n" if $media ne $oldmedia;
5093
5094             if (!drive_is_cdrom($old_drive)) {
5095
5096                 if ($drive->{file} ne $old_drive->{file}) {
5097
5098                     die "skip\n" if !$hotplug;
5099
5100                     # unplug and register as unused
5101                     vm_deviceunplug($vmid, $conf, $opt);
5102                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5103
5104                 } else {
5105                     # update existing disk
5106
5107                     # skip non hotpluggable value
5108                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5109                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5110                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5111                         &$safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5112                         &$safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5113                         die "skip\n";
5114                     }
5115
5116                     # apply throttle
5117                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5118                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5119                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5120                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5121                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5122                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5123                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5124                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5125                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5126                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5127                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5128                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5129                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5130                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5131                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5132                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5133                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5134                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5135
5136                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5137                                                    ($drive->{mbps} || 0)*1024*1024,
5138                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5139                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5140                                                    $drive->{iops} || 0,
5141                                                    $drive->{iops_rd} || 0,
5142                                                    $drive->{iops_wr} || 0,
5143                                                    ($drive->{mbps_max} || 0)*1024*1024,
5144                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5145                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5146                                                    $drive->{iops_max} || 0,
5147                                                    $drive->{iops_rd_max} || 0,
5148                                                    $drive->{iops_wr_max} || 0,
5149                                                    $drive->{bps_max_length} || 1,
5150                                                    $drive->{bps_rd_max_length} || 1,
5151                                                    $drive->{bps_wr_max_length} || 1,
5152                                                    $drive->{iops_max_length} || 1,
5153                                                    $drive->{iops_rd_max_length} || 1,
5154                                                    $drive->{iops_wr_max_length} || 1);
5155
5156                     }
5157
5158                     return 1;
5159                 }
5160
5161             } else { # cdrom
5162
5163                 if ($drive->{file} eq 'none') {
5164                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5165                     if (drive_is_cloudinit($old_drive)) {
5166                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5167                     }
5168                 } else {
5169                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5170
5171                     # force eject if locked
5172                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5173
5174                     if ($path) {
5175                         mon_cmd($vmid, "blockdev-change-medium",
5176                             id => "$opt", filename => "$path");
5177                     }
5178                 }
5179
5180                 return 1;
5181             }
5182         }
5183     }
5184
5185     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5186     # hotplug new disks
5187     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5188     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5189 }
5190
5191 sub vm_start {
5192     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5193         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage, $timeout) = @_;
5194
5195     PVE::QemuConfig->lock_config($vmid, sub {
5196         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5197
5198         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5199
5200         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5201
5202         PVE::QemuConfig->check_lock($conf)
5203             if !($skiplock || $is_suspended);
5204
5205         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5206
5207         # clean up leftover reboot request files
5208         eval { clear_reboot_request($vmid); };
5209         warn $@ if $@;
5210
5211         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5212             vmconfig_apply_pending($vmid, $conf, $storecfg);
5213             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5214         }
5215
5216         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5217
5218         my $defaults = load_defaults();
5219
5220         # set environment variable useful inside network script
5221         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5222
5223         my $local_volumes = {};
5224
5225         if ($targetstorage) {
5226             foreach_drive($conf, sub {
5227                 my ($ds, $drive) = @_;
5228
5229                 return if drive_is_cdrom($drive);
5230
5231                 my $volid = $drive->{file};
5232
5233                 return if !$volid;
5234
5235                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5236
5237                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5238                 return if $scfg->{shared};
5239                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5240             });
5241
5242             my $format = undef;
5243
5244             foreach my $opt (sort keys %$local_volumes) {
5245
5246                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5247                 my $drive = parse_drive($opt, $conf->{$opt});
5248
5249                 # If a remote storage is specified and the format of the original
5250                 # volume is not available there, fall back to the default format.
5251                 # Otherwise use the same format as the original.
5252                 if ($targetstorage && $targetstorage ne "1") {
5253                     $storeid = $targetstorage;
5254                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5255                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5256                     my $fileFormat = qemu_img_format($scfg, $volname);
5257                     $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5258                 } else {
5259                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5260                     $format = qemu_img_format($scfg, $volname);
5261                 }
5262
5263                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5264                 my $newdrive = $drive;
5265                 $newdrive->{format} = $format;
5266                 $newdrive->{file} = $newvolid;
5267                 my $drivestr = print_drive($newdrive);
5268                 $local_volumes->{$opt} = $drivestr;
5269                 #pass drive to conf for command line
5270                 $conf->{$opt} = $drivestr;
5271             }
5272         }
5273
5274         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5275
5276         if ($is_suspended) {
5277             # enforce machine type on suspended vm to ensure HW compatibility
5278             $forcemachine = $conf->{runningmachine};
5279             print "Resuming suspended VM\n";
5280         }
5281
5282         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5283
5284         my $migration_ip;
5285         my $get_migration_ip = sub {
5286             my ($cidr, $nodename) = @_;
5287
5288             return $migration_ip if defined($migration_ip);
5289
5290             if (!defined($cidr)) {
5291                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5292                 $cidr = $dc_conf->{migration}->{network};
5293             }
5294
5295             if (defined($cidr)) {
5296                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5297
5298                 die "could not get IP: no address configured on local " .
5299                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5300
5301                 die "could not get IP: multiple addresses configured on local " .
5302                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5303
5304                 $migration_ip = @$ips[0];
5305             }
5306
5307             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5308                 if !defined($migration_ip);
5309
5310             return $migration_ip;
5311         };
5312
5313         my $migrate_uri;
5314         if ($statefile) {
5315             if ($statefile eq 'tcp') {
5316                 my $localip = "localhost";
5317                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5318                 my $nodename = nodename();
5319
5320                 if (!defined($migration_type)) {
5321                     if (defined($datacenterconf->{migration}->{type})) {
5322                         $migration_type = $datacenterconf->{migration}->{type};
5323                     } else {
5324                         $migration_type = 'secure';
5325                     }
5326                 }
5327
5328                 if ($migration_type eq 'insecure') {
5329                     $localip = $get_migration_ip->($migration_network, $nodename);
5330                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5331                 }
5332
5333                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5334                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5335                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5336                 push @$cmd, '-incoming', $migrate_uri;
5337                 push @$cmd, '-S';
5338
5339             } elsif ($statefile eq 'unix') {
5340                 # should be default for secure migrations as a ssh TCP forward
5341                 # tunnel is not deterministic reliable ready and fails regurarly
5342                 # to set up in time, so use UNIX socket forwards
5343                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5344                 unlink $socket_addr;
5345
5346                 $migrate_uri = "unix:$socket_addr";
5347
5348                 push @$cmd, '-incoming', $migrate_uri;
5349                 push @$cmd, '-S';
5350
5351             } elsif (-e $statefile) {
5352                 push @$cmd, '-loadstate', $statefile;
5353             } else {
5354                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5355                 push @$vollist, $statefile;
5356                 push @$cmd, '-loadstate', $statepath;
5357             }
5358         } elsif ($paused) {
5359             push @$cmd, '-S';
5360         }
5361
5362         # host pci devices
5363         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5364           my $d = parse_hostpci($conf->{"hostpci$i"});
5365           next if !$d;
5366           my $pcidevices = $d->{pciid};
5367           foreach my $pcidevice (@$pcidevices) {
5368                 my $pciid = $pcidevice->{id};
5369
5370                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5371                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5372                 die "no pci device info for device '$pciid'\n" if !$info;
5373
5374                 if ($d->{mdev}) {
5375                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5376                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5377                 } else {
5378                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5379                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5380                     die "can't reset pci device '$pciid'\n"
5381                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5382                 }
5383           }
5384         }
5385
5386         PVE::Storage::activate_volumes($storecfg, $vollist);
5387
5388         eval {
5389             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5390                 outfunc => sub {}, errfunc => sub {});
5391         };
5392         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5393         # timeout should be more than enough here...
5394         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5395
5396         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5397                                                   : $defaults->{cpuunits};
5398
5399         my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
5400         my %run_params = (
5401             timeout => $statefile ? undef : $start_timeout,
5402             umask => 0077,
5403             noerr => 1,
5404         );
5405
5406         # when migrating, prefix QEMU output so other side can pick up any
5407         # errors that might occur and show the user
5408         if ($migratedfrom) {
5409             $run_params{quiet} = 1;
5410             $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5411         }
5412
5413         my %properties = (
5414             Slice => 'qemu.slice',
5415             KillMode => 'none',
5416             CPUShares => $cpuunits
5417         );
5418
5419         if (my $cpulimit = $conf->{cpulimit}) {
5420             $properties{CPUQuota} = int($cpulimit * 100);
5421         }
5422         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5423
5424         my $run_qemu = sub {
5425             PVE::Tools::run_fork sub {
5426                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5427
5428                 my $exitcode = run_command($cmd, %run_params);
5429                 die "QEMU exited with code $exitcode\n" if $exitcode;
5430             };
5431         };
5432
5433         if ($conf->{hugepages}) {
5434
5435             my $code = sub {
5436                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5437                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5438
5439                 PVE::QemuServer::Memory::hugepages_mount();
5440                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5441
5442                 eval { $run_qemu->() };
5443                 if (my $err = $@) {
5444                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5445                     die $err;
5446                 }
5447
5448                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5449             };
5450             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5451
5452         } else {
5453             eval { $run_qemu->() };
5454         }
5455
5456         if (my $err = $@) {
5457             # deactivate volumes if start fails
5458             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5459             die "start failed: $err";
5460         }
5461
5462         print "migration listens on $migrate_uri\n" if $migrate_uri;
5463
5464         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5465             eval { mon_cmd($vmid, "cont"); };
5466             warn $@ if $@;
5467         }
5468
5469         #start nbd server for storage migration
5470         if ($targetstorage) {
5471             my $nodename = nodename();
5472             my $localip = $get_migration_ip->($migration_network, $nodename);
5473             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5474             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5475
5476             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5477
5478             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5479
5480             foreach my $opt (sort keys %$local_volumes) {
5481                 my $drivestr = $local_volumes->{$opt};
5482                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5483                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5484                 print "storage migration listens on $migrate_storage_uri volume:$drivestr\n";
5485             }
5486         }
5487
5488         if ($migratedfrom) {
5489             eval {
5490                 set_migration_caps($vmid);
5491             };
5492             warn $@ if $@;
5493
5494             if ($spice_port) {
5495                 print "spice listens on port $spice_port\n";
5496                 if ($spice_ticket) {
5497                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5498                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5499                 }
5500             }
5501
5502         } else {
5503             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5504                 if !$statefile && $conf->{balloon};
5505
5506             foreach my $opt (keys %$conf) {
5507                 next if $opt !~  m/^net\d+$/;
5508                 my $nicconf = parse_net($conf->{$opt});
5509                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5510             }
5511         }
5512
5513         mon_cmd($vmid, 'qom-set',
5514                     path => "machine/peripheral/balloon0",
5515                     property => "guest-stats-polling-interval",
5516                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5517
5518         if ($is_suspended) {
5519             print "Resumed VM, removing state\n";
5520             if (my $vmstate = $conf->{vmstate}) {
5521                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5522                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5523             }
5524             delete $conf->@{qw(lock vmstate runningmachine)};
5525             PVE::QemuConfig->write_config($vmid, $conf);
5526         }
5527
5528         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5529     });
5530 }
5531
5532 sub vm_commandline {
5533     my ($storecfg, $vmid, $snapname) = @_;
5534
5535     my $conf = PVE::QemuConfig->load_config($vmid);
5536     my $forcemachine;
5537
5538     if ($snapname) {
5539         my $snapshot = $conf->{snapshots}->{$snapname};
5540         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5541
5542         # check for a 'runningmachine' in snapshot
5543         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5544
5545         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5546
5547         $conf = $snapshot;
5548     }
5549
5550     my $defaults = load_defaults();
5551
5552     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5553
5554     return PVE::Tools::cmd2string($cmd);
5555 }
5556
5557 sub vm_reset {
5558     my ($vmid, $skiplock) = @_;
5559
5560     PVE::QemuConfig->lock_config($vmid, sub {
5561
5562         my $conf = PVE::QemuConfig->load_config($vmid);
5563
5564         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5565
5566         mon_cmd($vmid, "system_reset");
5567     });
5568 }
5569
5570 sub get_vm_volumes {
5571     my ($conf) = @_;
5572
5573     my $vollist = [];
5574     foreach_volid($conf, sub {
5575         my ($volid, $attr) = @_;
5576
5577         return if $volid =~ m|^/|;
5578
5579         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5580         return if !$sid;
5581
5582         push @$vollist, $volid;
5583     });
5584
5585     return $vollist;
5586 }
5587
5588 sub vm_stop_cleanup {
5589     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5590
5591     eval {
5592
5593         if (!$keepActive) {
5594             my $vollist = get_vm_volumes($conf);
5595             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5596         }
5597
5598         foreach my $ext (qw(mon qmp pid vnc qga)) {
5599             unlink "/var/run/qemu-server/${vmid}.$ext";
5600         }
5601
5602         if ($conf->{ivshmem}) {
5603             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5604             # just delete it for now, VMs which have this already open do not
5605             # are affected, but new VMs will get a separated one. If this
5606             # becomes an issue we either add some sort of ref-counting or just
5607             # add a "don't delete on stop" flag to the ivshmem format.
5608             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5609         }
5610
5611         foreach my $key (keys %$conf) {
5612             next if $key !~ m/^hostpci(\d+)$/;
5613             my $hostpciindex = $1;
5614             my $d = parse_hostpci($conf->{$key});
5615             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5616
5617             foreach my $pci (@{$d->{pciid}}) {
5618                 my $pciid = $pci->{id};
5619                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5620             }
5621         }
5622
5623         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5624     };
5625     warn $@ if $@; # avoid errors - just warn
5626 }
5627
5628 # call only in locked context
5629 sub _do_vm_stop {
5630     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5631
5632     my $pid = check_running($vmid, $nocheck);
5633     return if !$pid;
5634
5635     my $conf;
5636     if (!$nocheck) {
5637         $conf = PVE::QemuConfig->load_config($vmid);
5638         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5639         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5640             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5641             $timeout = $opts->{down} if $opts->{down};
5642         }
5643         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5644     }
5645
5646     eval {
5647         if ($shutdown) {
5648             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5649                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5650             } else {
5651                 mon_cmd($vmid, "system_powerdown");
5652             }
5653         } else {
5654             mon_cmd($vmid, "quit");
5655         }
5656     };
5657     my $err = $@;
5658
5659     if (!$err) {
5660         $timeout = 60 if !defined($timeout);
5661
5662         my $count = 0;
5663         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5664             $count++;
5665             sleep 1;
5666         }
5667
5668         if ($count >= $timeout) {
5669             if ($force) {
5670                 warn "VM still running - terminating now with SIGTERM\n";
5671                 kill 15, $pid;
5672             } else {
5673                 die "VM quit/powerdown failed - got timeout\n";
5674             }
5675         } else {
5676             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5677             return;
5678         }
5679     } else {
5680         if ($force) {
5681             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5682             kill 15, $pid;
5683         } else {
5684             die "VM quit/powerdown failed\n";
5685         }
5686     }
5687
5688     # wait again
5689     $timeout = 10;
5690
5691     my $count = 0;
5692     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5693         $count++;
5694         sleep 1;
5695     }
5696
5697     if ($count >= $timeout) {
5698         warn "VM still running - terminating now with SIGKILL\n";
5699         kill 9, $pid;
5700         sleep 1;
5701     }
5702
5703     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5704 }
5705
5706 # Note: use $nocheck to skip tests if VM configuration file exists.
5707 # We need that when migration VMs to other nodes (files already moved)
5708 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5709 sub vm_stop {
5710     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5711
5712     $force = 1 if !defined($force) && !$shutdown;
5713
5714     if ($migratedfrom){
5715         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5716         kill 15, $pid if $pid;
5717         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5718         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5719         return;
5720     }
5721
5722     PVE::QemuConfig->lock_config($vmid, sub {
5723         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5724    });
5725 }
5726
5727 sub vm_reboot {
5728     my ($vmid, $timeout) = @_;
5729
5730     PVE::QemuConfig->lock_config($vmid, sub {
5731         eval {
5732
5733             # only reboot if running, as qmeventd starts it again on a stop event
5734             return if !check_running($vmid);
5735
5736             create_reboot_request($vmid);
5737
5738             my $storecfg = PVE::Storage::config();
5739             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5740
5741         };
5742         if (my $err = $@) {
5743             # avoid that the next normal shutdown will be confused for a reboot
5744             clear_reboot_request($vmid);
5745             die $err;
5746         }
5747    });
5748 }
5749
5750 # note: if using the statestorage parameter, the caller has to check privileges
5751 sub vm_suspend {
5752     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5753
5754     my $conf;
5755     my $path;
5756     my $storecfg;
5757     my $vmstate;
5758
5759     PVE::QemuConfig->lock_config($vmid, sub {
5760
5761         $conf = PVE::QemuConfig->load_config($vmid);
5762
5763         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5764         PVE::QemuConfig->check_lock($conf)
5765             if !($skiplock || $is_backing_up);
5766
5767         die "cannot suspend to disk during backup\n"
5768             if $is_backing_up && $includestate;
5769
5770         if ($includestate) {
5771             $conf->{lock} = 'suspending';
5772             my $date = strftime("%Y-%m-%d", localtime(time()));
5773             $storecfg = PVE::Storage::config();
5774             if (!$statestorage) {
5775                 $statestorage = find_vmstate_storage($conf, $storecfg);
5776                 # check permissions for the storage
5777                 my $rpcenv = PVE::RPCEnvironment::get();
5778                 if ($rpcenv->{type} ne 'cli') {
5779                     my $authuser = $rpcenv->get_user();
5780                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5781                 }
5782             }
5783
5784
5785             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5786             $path = PVE::Storage::path($storecfg, $vmstate);
5787             PVE::QemuConfig->write_config($vmid, $conf);
5788         } else {
5789             mon_cmd($vmid, "stop");
5790         }
5791     });
5792
5793     if ($includestate) {
5794         # save vm state
5795         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5796
5797         eval {
5798             mon_cmd($vmid, "savevm-start", statefile => $path);
5799             for(;;) {
5800                 my $state = mon_cmd($vmid, "query-savevm");
5801                 if (!$state->{status}) {
5802                     die "savevm not active\n";
5803                 } elsif ($state->{status} eq 'active') {
5804                     sleep(1);
5805                     next;
5806                 } elsif ($state->{status} eq 'completed') {
5807                     print "State saved, quitting\n";
5808                     last;
5809                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5810                     die "query-savevm failed with error '$state->{error}'\n"
5811                 } else {
5812                     die "query-savevm returned status '$state->{status}'\n";
5813                 }
5814             }
5815         };
5816         my $err = $@;
5817
5818         PVE::QemuConfig->lock_config($vmid, sub {
5819             $conf = PVE::QemuConfig->load_config($vmid);
5820             if ($err) {
5821                 # cleanup, but leave suspending lock, to indicate something went wrong
5822                 eval {
5823                     mon_cmd($vmid, "savevm-end");
5824                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5825                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5826                     delete $conf->@{qw(vmstate runningmachine)};
5827                     PVE::QemuConfig->write_config($vmid, $conf);
5828                 };
5829                 warn $@ if $@;
5830                 die $err;
5831             }
5832
5833             die "lock changed unexpectedly\n"
5834                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5835
5836             mon_cmd($vmid, "quit");
5837             $conf->{lock} = 'suspended';
5838             PVE::QemuConfig->write_config($vmid, $conf);
5839         });
5840     }
5841 }
5842
5843 sub vm_resume {
5844     my ($vmid, $skiplock, $nocheck) = @_;
5845
5846     PVE::QemuConfig->lock_config($vmid, sub {
5847         my $res = mon_cmd($vmid, 'query-status');
5848         my $resume_cmd = 'cont';
5849
5850         if ($res->{status} && $res->{status} eq 'suspended') {
5851             $resume_cmd = 'system_wakeup';
5852         }
5853
5854         if (!$nocheck) {
5855
5856             my $conf = PVE::QemuConfig->load_config($vmid);
5857
5858             PVE::QemuConfig->check_lock($conf)
5859                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5860         }
5861
5862         mon_cmd($vmid, $resume_cmd);
5863     });
5864 }
5865
5866 sub vm_sendkey {
5867     my ($vmid, $skiplock, $key) = @_;
5868
5869     PVE::QemuConfig->lock_config($vmid, sub {
5870
5871         my $conf = PVE::QemuConfig->load_config($vmid);
5872
5873         # there is no qmp command, so we use the human monitor command
5874         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5875         die $res if $res ne '';
5876     });
5877 }
5878
5879 # vzdump restore implementaion
5880
5881 sub tar_archive_read_firstfile {
5882     my $archive = shift;
5883
5884     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5885
5886     # try to detect archive type first
5887     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5888         die "unable to open file '$archive'\n";
5889     my $firstfile = <$fh>;
5890     kill 15, $pid;
5891     close $fh;
5892
5893     die "ERROR: archive contaions no data\n" if !$firstfile;
5894     chomp $firstfile;
5895
5896     return $firstfile;
5897 }
5898
5899 sub tar_restore_cleanup {
5900     my ($storecfg, $statfile) = @_;
5901
5902     print STDERR "starting cleanup\n";
5903
5904     if (my $fd = IO::File->new($statfile, "r")) {
5905         while (defined(my $line = <$fd>)) {
5906             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5907                 my $volid = $2;
5908                 eval {
5909                     if ($volid =~ m|^/|) {
5910                         unlink $volid || die 'unlink failed\n';
5911                     } else {
5912                         PVE::Storage::vdisk_free($storecfg, $volid);
5913                     }
5914                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5915                 };
5916                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5917             } else {
5918                 print STDERR "unable to parse line in statfile - $line";
5919             }
5920         }
5921         $fd->close();
5922     }
5923 }
5924
5925 sub restore_archive {
5926     my ($archive, $vmid, $user, $opts) = @_;
5927
5928     my $format = $opts->{format};
5929     my $comp;
5930
5931     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5932         $format = 'tar' if !$format;
5933         $comp = 'gzip';
5934     } elsif ($archive =~ m/\.tar$/) {
5935         $format = 'tar' if !$format;
5936     } elsif ($archive =~ m/.tar.lzo$/) {
5937         $format = 'tar' if !$format;
5938         $comp = 'lzop';
5939     } elsif ($archive =~ m/\.vma$/) {
5940         $format = 'vma' if !$format;
5941     } elsif ($archive =~ m/\.vma\.gz$/) {
5942         $format = 'vma' if !$format;
5943         $comp = 'gzip';
5944     } elsif ($archive =~ m/\.vma\.lzo$/) {
5945         $format = 'vma' if !$format;
5946         $comp = 'lzop';
5947     } else {
5948         $format = 'vma' if !$format; # default
5949     }
5950
5951     # try to detect archive format
5952     if ($format eq 'tar') {
5953         return restore_tar_archive($archive, $vmid, $user, $opts);
5954     } else {
5955         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5956     }
5957 }
5958
5959 sub restore_update_config_line {
5960     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5961
5962     return if $line =~ m/^\#qmdump\#/;
5963     return if $line =~ m/^\#vzdump\#/;
5964     return if $line =~ m/^lock:/;
5965     return if $line =~ m/^unused\d+:/;
5966     return if $line =~ m/^parent:/;
5967
5968     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5969     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5970         # try to convert old 1.X settings
5971         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5972         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5973             my ($model, $macaddr) = split(/\=/, $devconfig);
5974             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5975             my $net = {
5976                 model => $model,
5977                 bridge => "vmbr$ind",
5978                 macaddr => $macaddr,
5979             };
5980             my $netstr = print_net($net);
5981
5982             print $outfd "net$cookie->{netcount}: $netstr\n";
5983             $cookie->{netcount}++;
5984         }
5985     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5986         my ($id, $netstr) = ($1, $2);
5987         my $net = parse_net($netstr);
5988         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5989         $netstr = print_net($net);
5990         print $outfd "$id: $netstr\n";
5991     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5992         my $virtdev = $1;
5993         my $value = $3;
5994         my $di = parse_drive($virtdev, $value);
5995         if (defined($di->{backup}) && !$di->{backup}) {
5996             print $outfd "#$line";
5997         } elsif ($map->{$virtdev}) {
5998             delete $di->{format}; # format can change on restore
5999             $di->{file} = $map->{$virtdev};
6000             $value = print_drive($di);
6001             print $outfd "$virtdev: $value\n";
6002         } else {
6003             print $outfd $line;
6004         }
6005     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6006         my $vmgenid = $1;
6007         if ($vmgenid ne '0') {
6008             # always generate a new vmgenid if there was a valid one setup
6009             $vmgenid = generate_uuid();
6010         }
6011         print $outfd "vmgenid: $vmgenid\n";
6012     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6013         my ($uuid, $uuid_str);
6014         UUID::generate($uuid);
6015         UUID::unparse($uuid, $uuid_str);
6016         my $smbios1 = parse_smbios1($2);
6017         $smbios1->{uuid} = $uuid_str;
6018         print $outfd $1.print_smbios1($smbios1)."\n";
6019     } else {
6020         print $outfd $line;
6021     }
6022 }
6023
6024 sub scan_volids {
6025     my ($cfg, $vmid) = @_;
6026
6027     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6028
6029     my $volid_hash = {};
6030     foreach my $storeid (keys %$info) {
6031         foreach my $item (@{$info->{$storeid}}) {
6032             next if !($item->{volid} && $item->{size});
6033             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6034             $volid_hash->{$item->{volid}} = $item;
6035         }
6036     }
6037
6038     return $volid_hash;
6039 }
6040
6041 sub is_volume_in_use {
6042     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6043
6044     my $path = PVE::Storage::path($storecfg, $volid);
6045
6046     my $scan_config = sub {
6047         my ($cref, $snapname) = @_;
6048
6049         foreach my $key (keys %$cref) {
6050             my $value = $cref->{$key};
6051             if (is_valid_drivename($key)) {
6052                 next if $skip_drive && $key eq $skip_drive;
6053                 my $drive = parse_drive($key, $value);
6054                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6055                 return 1 if $volid eq $drive->{file};
6056                 if ($drive->{file} =~ m!^/!) {
6057                     return 1 if $drive->{file} eq $path;
6058                 } else {
6059                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6060                     next if !$storeid;
6061                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6062                     next if !$scfg;
6063                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6064                 }
6065             }
6066         }
6067
6068         return 0;
6069     };
6070
6071     return 1 if &$scan_config($conf);
6072
6073     undef $skip_drive;
6074
6075     foreach my $snapname (keys %{$conf->{snapshots}}) {
6076         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6077     }
6078
6079     return 0;
6080 }
6081
6082 sub update_disksize {
6083     my ($drive, $volid_hash) = @_;
6084
6085     my $volid = $drive->{file};
6086     return undef if !defined($volid);
6087
6088     my $oldsize = $drive->{size};
6089     my $newsize = $volid_hash->{$volid}->{size};
6090
6091     if (defined($newsize) && defined($oldsize) && $newsize != $oldsize) {
6092         $drive->{size} = $newsize;
6093
6094         my $old_fmt = PVE::JSONSchema::format_size($oldsize);
6095         my $new_fmt = PVE::JSONSchema::format_size($newsize);
6096
6097         return wantarray ? ($drive, $old_fmt, $new_fmt) : $drive;
6098     }
6099
6100     return undef;
6101 }
6102
6103 sub update_disk_config {
6104     my ($vmid, $conf, $volid_hash) = @_;
6105
6106     my $changes;
6107     my $prefix = "VM $vmid:";
6108
6109     # used and unused disks
6110     my $referenced = {};
6111
6112     # Note: it is allowed to define multiple storages with same path (alias), so
6113     # we need to check both 'volid' and real 'path' (two different volid can point
6114     # to the same path).
6115
6116     my $referencedpath = {};
6117
6118     # update size info
6119     foreach my $opt (keys %$conf) {
6120         if (is_valid_drivename($opt)) {
6121             my $drive = parse_drive($opt, $conf->{$opt});
6122             my $volid = $drive->{file};
6123             next if !$volid;
6124
6125             # mark volid as "in-use" for next step
6126             $referenced->{$volid} = 1;
6127             if ($volid_hash->{$volid} &&
6128                 (my $path = $volid_hash->{$volid}->{path})) {
6129                 $referencedpath->{$path} = 1;
6130             }
6131
6132             next if drive_is_cdrom($drive);
6133             next if !$volid_hash->{$volid};
6134
6135             my ($updated, $old_size, $new_size) = update_disksize($drive, $volid_hash);
6136             if (defined($updated)) {
6137                 $changes = 1;
6138                 $conf->{$opt} = print_drive($updated);
6139                 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
6140             }
6141         }
6142     }
6143
6144     # remove 'unusedX' entry if volume is used
6145     foreach my $opt (keys %$conf) {
6146         next if $opt !~ m/^unused\d+$/;
6147         my $volid = $conf->{$opt};
6148         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6149         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6150             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6151             $changes = 1;
6152             delete $conf->{$opt};
6153         }
6154
6155         $referenced->{$volid} = 1;
6156         $referencedpath->{$path} = 1 if $path;
6157     }
6158
6159     foreach my $volid (sort keys %$volid_hash) {
6160         next if $volid =~ m/vm-$vmid-state-/;
6161         next if $referenced->{$volid};
6162         my $path = $volid_hash->{$volid}->{path};
6163         next if !$path; # just to be sure
6164         next if $referencedpath->{$path};
6165         $changes = 1;
6166         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6167         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6168         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6169     }
6170
6171     return $changes;
6172 }
6173
6174 sub rescan {
6175     my ($vmid, $nolock, $dryrun) = @_;
6176
6177     my $cfg = PVE::Storage::config();
6178
6179     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6180     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6181     foreach my $stor (keys %{$cfg->{ids}}) {
6182         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6183     }
6184
6185     print "rescan volumes...\n";
6186     my $volid_hash = scan_volids($cfg, $vmid);
6187
6188     my $updatefn =  sub {
6189         my ($vmid) = @_;
6190
6191         my $conf = PVE::QemuConfig->load_config($vmid);
6192
6193         PVE::QemuConfig->check_lock($conf);
6194
6195         my $vm_volids = {};
6196         foreach my $volid (keys %$volid_hash) {
6197             my $info = $volid_hash->{$volid};
6198             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6199         }
6200
6201         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6202
6203         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6204     };
6205
6206     if (defined($vmid)) {
6207         if ($nolock) {
6208             &$updatefn($vmid);
6209         } else {
6210             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6211         }
6212     } else {
6213         my $vmlist = config_list();
6214         foreach my $vmid (keys %$vmlist) {
6215             if ($nolock) {
6216                 &$updatefn($vmid);
6217             } else {
6218                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6219             }
6220         }
6221     }
6222 }
6223
6224 sub restore_vma_archive {
6225     my ($archive, $vmid, $user, $opts, $comp) = @_;
6226
6227     my $readfrom = $archive;
6228
6229     my $cfg = PVE::Storage::config();
6230     my $commands = [];
6231     my $bwlimit = $opts->{bwlimit};
6232
6233     my $dbg_cmdstring = '';
6234     my $add_pipe = sub {
6235         my ($cmd) = @_;
6236         push @$commands, $cmd;
6237         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6238         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6239         $readfrom = '-';
6240     };
6241
6242     my $input = undef;
6243     if ($archive eq '-') {
6244         $input = '<&STDIN';
6245     } else {
6246         # If we use a backup from a PVE defined storage we also consider that
6247         # storage's rate limit:
6248         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6249         if (defined($volid)) {
6250             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6251             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6252             if ($readlimit) {
6253                 print STDERR "applying read rate limit: $readlimit\n";
6254                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6255                 $add_pipe->($cstream);
6256             }
6257         }
6258     }
6259
6260     if ($comp) {
6261         my $cmd;
6262         if ($comp eq 'gzip') {
6263             $cmd = ['zcat', $readfrom];
6264         } elsif ($comp eq 'lzop') {
6265             $cmd = ['lzop', '-d', '-c', $readfrom];
6266         } else {
6267             die "unknown compression method '$comp'\n";
6268         }
6269         $add_pipe->($cmd);
6270     }
6271
6272     my $tmpdir = "/var/tmp/vzdumptmp$$";
6273     rmtree $tmpdir;
6274
6275     # disable interrupts (always do cleanups)
6276     local $SIG{INT} =
6277         local $SIG{TERM} =
6278         local $SIG{QUIT} =
6279         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6280
6281     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6282     POSIX::mkfifo($mapfifo, 0600);
6283     my $fifofh;
6284
6285     my $openfifo = sub {
6286         open($fifofh, '>', $mapfifo) || die $!;
6287     };
6288
6289     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6290
6291     my $oldtimeout;
6292     my $timeout = 5;
6293
6294     my $devinfo = {};
6295
6296     my $rpcenv = PVE::RPCEnvironment::get();
6297
6298     my $conffile = PVE::QemuConfig->config_file($vmid);
6299     my $tmpfn = "$conffile.$$.tmp";
6300
6301     # Note: $oldconf is undef if VM does not exist
6302     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6303     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6304
6305     my %storage_limits;
6306
6307     my $print_devmap = sub {
6308         my $virtdev_hash = {};
6309
6310         my $cfgfn = "$tmpdir/qemu-server.conf";
6311
6312         # we can read the config - that is already extracted
6313         my $fh = IO::File->new($cfgfn, "r") ||
6314             "unable to read qemu-server.conf - $!\n";
6315
6316         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6317         if (-f $fwcfgfn) {
6318             my $pve_firewall_dir = '/etc/pve/firewall';
6319             mkdir $pve_firewall_dir; # make sure the dir exists
6320             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6321         }
6322
6323         while (defined(my $line = <$fh>)) {
6324             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6325                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6326                 die "archive does not contain data for drive '$virtdev'\n"
6327                     if !$devinfo->{$devname};
6328                 if (defined($opts->{storage})) {
6329                     $storeid = $opts->{storage} || 'local';
6330                 } elsif (!$storeid) {
6331                     $storeid = 'local';
6332                 }
6333                 $format = 'raw' if !$format;
6334                 $devinfo->{$devname}->{devname} = $devname;
6335                 $devinfo->{$devname}->{virtdev} = $virtdev;
6336                 $devinfo->{$devname}->{format} = $format;
6337                 $devinfo->{$devname}->{storeid} = $storeid;
6338
6339                 # check permission on storage
6340                 my $pool = $opts->{pool}; # todo: do we need that?
6341                 if ($user ne 'root@pam') {
6342                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6343                 }
6344
6345                 $storage_limits{$storeid} = $bwlimit;
6346
6347                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6348             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6349                 my $virtdev = $1;
6350                 my $drive = parse_drive($virtdev, $2);
6351                 if (drive_is_cloudinit($drive)) {
6352                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6353                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6354                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6355
6356                     my $d = {
6357                         format => $format,
6358                         storeid => $opts->{storage} // $storeid,
6359                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6360                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6361                         name => "vm-$vmid-cloudinit",
6362                         is_cloudinit => 1,
6363                     };
6364                     $virtdev_hash->{$virtdev} = $d;
6365                 }
6366             }
6367         }
6368
6369         foreach my $key (keys %storage_limits) {
6370             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6371             next if !$limit;
6372             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6373             $storage_limits{$key} = $limit * 1024;
6374         }
6375
6376         foreach my $devname (keys %$devinfo) {
6377             die "found no device mapping information for device '$devname'\n"
6378                 if !$devinfo->{$devname}->{virtdev};
6379         }
6380
6381         # create empty/temp config
6382         if ($oldconf) {
6383             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6384             foreach_drive($oldconf, sub {
6385                 my ($ds, $drive) = @_;
6386
6387                 return if drive_is_cdrom($drive, 1);
6388
6389                 my $volid = $drive->{file};
6390                 return if !$volid || $volid =~ m|^/|;
6391
6392                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6393                 return if !$path || !$owner || ($owner != $vmid);
6394
6395                 # Note: only delete disk we want to restore
6396                 # other volumes will become unused
6397                 if ($virtdev_hash->{$ds}) {
6398                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6399                     if (my $err = $@) {
6400                         warn $err;
6401                     }
6402                 }
6403             });
6404
6405             # delete vmstate files, after the restore we have no snapshots anymore
6406             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6407                 my $snap = $oldconf->{snapshots}->{$snapname};
6408                 if ($snap->{vmstate}) {
6409                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6410                     if (my $err = $@) {
6411                         warn $err;
6412                     }
6413                 }
6414             }
6415         }
6416
6417         my $map = {};
6418         foreach my $virtdev (sort keys %$virtdev_hash) {
6419             my $d = $virtdev_hash->{$virtdev};
6420             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6421             my $storeid = $d->{storeid};
6422             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6423
6424             my $map_opts = '';
6425             if (my $limit = $storage_limits{$storeid}) {
6426                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6427             }
6428
6429             # test if requested format is supported
6430             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6431             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6432             $d->{format} = $defFormat if !$supported;
6433
6434             my $name;
6435             if ($d->{is_cloudinit}) {
6436                 $name = $d->{name};
6437                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6438             }
6439
6440             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6441             print STDERR "new volume ID is '$volid'\n";
6442             $d->{volid} = $volid;
6443
6444             PVE::Storage::activate_volumes($cfg, [$volid]);
6445
6446             my $write_zeros = 1;
6447             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6448                 $write_zeros = 0;
6449             }
6450
6451             if (!$d->{is_cloudinit}) {
6452                 my $path = PVE::Storage::path($cfg, $volid);
6453
6454                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6455
6456                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6457             }
6458             $map->{$virtdev} = $volid;
6459         }
6460
6461         $fh->seek(0, 0) || die "seek failed - $!\n";
6462
6463         my $outfd = new IO::File ($tmpfn, "w") ||
6464             die "unable to write config for VM $vmid\n";
6465
6466         my $cookie = { netcount => 0 };
6467         while (defined(my $line = <$fh>)) {
6468             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6469         }
6470
6471         $fh->close();
6472         $outfd->close();
6473     };
6474
6475     eval {
6476         # enable interrupts
6477         local $SIG{INT} =
6478             local $SIG{TERM} =
6479             local $SIG{QUIT} =
6480             local $SIG{HUP} =
6481             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6482         local $SIG{ALRM} = sub { die "got timeout\n"; };
6483
6484         $oldtimeout = alarm($timeout);
6485
6486         my $parser = sub {
6487             my $line = shift;
6488
6489             print "$line\n";
6490
6491             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6492                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6493                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6494             } elsif ($line =~ m/^CTIME: /) {
6495                 # we correctly received the vma config, so we can disable
6496                 # the timeout now for disk allocation (set to 10 minutes, so
6497                 # that we always timeout if something goes wrong)
6498                 alarm(600);
6499                 &$print_devmap();
6500                 print $fifofh "done\n";
6501                 my $tmp = $oldtimeout || 0;
6502                 $oldtimeout = undef;
6503                 alarm($tmp);
6504                 close($fifofh);
6505             }
6506         };
6507
6508         print "restore vma archive: $dbg_cmdstring\n";
6509         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6510     };
6511     my $err = $@;
6512
6513     alarm($oldtimeout) if $oldtimeout;
6514
6515     my $vollist = [];
6516     foreach my $devname (keys %$devinfo) {
6517         my $volid = $devinfo->{$devname}->{volid};
6518         push @$vollist, $volid if $volid;
6519     }
6520
6521     PVE::Storage::deactivate_volumes($cfg, $vollist);
6522
6523     unlink $mapfifo;
6524
6525     if ($err) {
6526         rmtree $tmpdir;
6527         unlink $tmpfn;
6528
6529         foreach my $devname (keys %$devinfo) {
6530             my $volid = $devinfo->{$devname}->{volid};
6531             next if !$volid;
6532             eval {
6533                 if ($volid =~ m|^/|) {
6534                     unlink $volid || die 'unlink failed\n';
6535                 } else {
6536                     PVE::Storage::vdisk_free($cfg, $volid);
6537                 }
6538                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6539             };
6540             print STDERR "unable to cleanup '$volid' - $@" if $@;
6541         }
6542         die $err;
6543     }
6544
6545     rmtree $tmpdir;
6546
6547     rename($tmpfn, $conffile) ||
6548         die "unable to commit configuration file '$conffile'\n";
6549
6550     PVE::Cluster::cfs_update(); # make sure we read new file
6551
6552     eval { rescan($vmid, 1); };
6553     warn $@ if $@;
6554 }
6555
6556 sub restore_tar_archive {
6557     my ($archive, $vmid, $user, $opts) = @_;
6558
6559     if ($archive ne '-') {
6560         my $firstfile = tar_archive_read_firstfile($archive);
6561         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6562             if $firstfile ne 'qemu-server.conf';
6563     }
6564
6565     my $storecfg = PVE::Storage::config();
6566
6567     # avoid zombie disks when restoring over an existing VM -> cleanup first
6568     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6569     # skiplock=1 because qmrestore has set the 'create' lock itself already
6570     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6571     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6572
6573     my $tocmd = "/usr/lib/qemu-server/qmextract";
6574
6575     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6576     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6577     $tocmd .= ' --prealloc' if $opts->{prealloc};
6578     $tocmd .= ' --info' if $opts->{info};
6579
6580     # tar option "xf" does not autodetect compression when read from STDIN,
6581     # so we pipe to zcat
6582     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6583         PVE::Tools::shellquote("--to-command=$tocmd");
6584
6585     my $tmpdir = "/var/tmp/vzdumptmp$$";
6586     mkpath $tmpdir;
6587
6588     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6589     local $ENV{VZDUMP_VMID} = $vmid;
6590     local $ENV{VZDUMP_USER} = $user;
6591
6592     my $conffile = PVE::QemuConfig->config_file($vmid);
6593     my $tmpfn = "$conffile.$$.tmp";
6594
6595     # disable interrupts (always do cleanups)
6596     local $SIG{INT} =
6597         local $SIG{TERM} =
6598         local $SIG{QUIT} =
6599         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6600
6601     eval {
6602         # enable interrupts
6603         local $SIG{INT} =
6604             local $SIG{TERM} =
6605             local $SIG{QUIT} =
6606             local $SIG{HUP} =
6607             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6608
6609         if ($archive eq '-') {
6610             print "extracting archive from STDIN\n";
6611             run_command($cmd, input => "<&STDIN");
6612         } else {
6613             print "extracting archive '$archive'\n";
6614             run_command($cmd);
6615         }
6616
6617         return if $opts->{info};
6618
6619         # read new mapping
6620         my $map = {};
6621         my $statfile = "$tmpdir/qmrestore.stat";
6622         if (my $fd = IO::File->new($statfile, "r")) {
6623             while (defined (my $line = <$fd>)) {
6624                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6625                     $map->{$1} = $2 if $1;
6626                 } else {
6627                     print STDERR "unable to parse line in statfile - $line\n";
6628                 }
6629             }
6630             $fd->close();
6631         }
6632
6633         my $confsrc = "$tmpdir/qemu-server.conf";
6634
6635         my $srcfd = new IO::File($confsrc, "r") ||
6636             die "unable to open file '$confsrc'\n";
6637
6638         my $outfd = new IO::File ($tmpfn, "w") ||
6639             die "unable to write config for VM $vmid\n";
6640
6641         my $cookie = { netcount => 0 };
6642         while (defined (my $line = <$srcfd>)) {
6643             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6644         }
6645
6646         $srcfd->close();
6647         $outfd->close();
6648     };
6649     if (my $err = $@) {
6650         unlink $tmpfn;
6651         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6652         die $err;
6653     }
6654
6655     rmtree $tmpdir;
6656
6657     rename $tmpfn, $conffile ||
6658         die "unable to commit configuration file '$conffile'\n";
6659
6660     PVE::Cluster::cfs_update(); # make sure we read new file
6661
6662     eval { rescan($vmid, 1); };
6663     warn $@ if $@;
6664 };
6665
6666 sub foreach_storage_used_by_vm {
6667     my ($conf, $func) = @_;
6668
6669     my $sidhash = {};
6670
6671     foreach_drive($conf, sub {
6672         my ($ds, $drive) = @_;
6673         return if drive_is_cdrom($drive);
6674
6675         my $volid = $drive->{file};
6676
6677         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6678         $sidhash->{$sid} = $sid if $sid;
6679     });
6680
6681     foreach my $sid (sort keys %$sidhash) {
6682         &$func($sid);
6683     }
6684 }
6685
6686 my $qemu_snap_storage = {
6687     rbd => 1,
6688 };
6689 sub do_snapshots_with_qemu {
6690     my ($storecfg, $volid) = @_;
6691
6692     my $storage_name = PVE::Storage::parse_volume_id($volid);
6693     my $scfg = $storecfg->{ids}->{$storage_name};
6694
6695     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6696         return 1;
6697     }
6698
6699     if ($volid =~ m/\.(qcow2|qed)$/){
6700         return 1;
6701     }
6702
6703     return undef;
6704 }
6705
6706 sub qga_check_running {
6707     my ($vmid, $nowarn) = @_;
6708
6709     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6710     if ($@) {
6711         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6712         return 0;
6713     }
6714     return 1;
6715 }
6716
6717 sub template_create {
6718     my ($vmid, $conf, $disk) = @_;
6719
6720     my $storecfg = PVE::Storage::config();
6721
6722     foreach_drive($conf, sub {
6723         my ($ds, $drive) = @_;
6724
6725         return if drive_is_cdrom($drive);
6726         return if $disk && $ds ne $disk;
6727
6728         my $volid = $drive->{file};
6729         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6730
6731         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6732         $drive->{file} = $voliddst;
6733         $conf->{$ds} = print_drive($drive);
6734         PVE::QemuConfig->write_config($vmid, $conf);
6735     });
6736 }
6737
6738 sub convert_iscsi_path {
6739     my ($path) = @_;
6740
6741     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6742         my $portal = $1;
6743         my $target = $2;
6744         my $lun = $3;
6745
6746         my $initiator_name = get_initiator_name();
6747
6748         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6749                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6750     }
6751
6752     die "cannot convert iscsi path '$path', unkown format\n";
6753 }
6754
6755 sub qemu_img_convert {
6756     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6757
6758     my $storecfg = PVE::Storage::config();
6759     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6760     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6761
6762     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6763
6764     my $cachemode;
6765     my $src_path;
6766     my $src_is_iscsi = 0;
6767     my $src_format;
6768
6769     if ($src_storeid) {
6770         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6771         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6772         $src_format = qemu_img_format($src_scfg, $src_volname);
6773         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6774         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6775         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6776     } elsif (-f $src_volid) {
6777         $src_path = $src_volid;
6778         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6779             $src_format = $1;
6780         }
6781     }
6782
6783     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6784
6785     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6786     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6787     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6788     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6789
6790     my $cmd = [];
6791     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6792     push @$cmd, '-l', "snapshot.name=$snapname"
6793         if $snapname && $src_format && $src_format eq "qcow2";
6794     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6795     push @$cmd, '-T', $cachemode if defined($cachemode);
6796
6797     if ($src_is_iscsi) {
6798         push @$cmd, '--image-opts';
6799         $src_path = convert_iscsi_path($src_path);
6800     } elsif ($src_format) {
6801         push @$cmd, '-f', $src_format;
6802     }
6803
6804     if ($dst_is_iscsi) {
6805         push @$cmd, '--target-image-opts';
6806         $dst_path = convert_iscsi_path($dst_path);
6807     } else {
6808         push @$cmd, '-O', $dst_format;
6809     }
6810
6811     push @$cmd, $src_path;
6812
6813     if (!$dst_is_iscsi && $is_zero_initialized) {
6814         push @$cmd, "zeroinit:$dst_path";
6815     } else {
6816         push @$cmd, $dst_path;
6817     }
6818
6819     my $parser = sub {
6820         my $line = shift;
6821         if($line =~ m/\((\S+)\/100\%\)/){
6822             my $percent = $1;
6823             my $transferred = int($size * $percent / 100);
6824             my $remaining = $size - $transferred;
6825
6826             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6827         }
6828
6829     };
6830
6831     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6832     my $err = $@;
6833     die "copy failed: $err" if $err;
6834 }
6835
6836 sub qemu_img_format {
6837     my ($scfg, $volname) = @_;
6838
6839     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6840         return $1;
6841     } else {
6842         return "raw";
6843     }
6844 }
6845
6846 sub qemu_drive_mirror {
6847     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6848
6849     $jobs = {} if !$jobs;
6850
6851     my $qemu_target;
6852     my $format;
6853     $jobs->{"drive-$drive"} = {};
6854
6855     if ($dst_volid =~ /^nbd:/) {
6856         $qemu_target = $dst_volid;
6857         $format = "nbd";
6858     } else {
6859         my $storecfg = PVE::Storage::config();
6860         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6861
6862         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6863
6864         $format = qemu_img_format($dst_scfg, $dst_volname);
6865
6866         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6867
6868         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6869     }
6870
6871     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6872     $opts->{format} = $format if $format;
6873
6874     if (defined($bwlimit)) {
6875         $opts->{speed} = $bwlimit * 1024;
6876         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6877     } else {
6878         print "drive mirror is starting for drive-$drive\n";
6879     }
6880
6881     # if a job already runs for this device we get an error, catch it for cleanup
6882     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6883     if (my $err = $@) {
6884         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6885         warn "$@\n" if $@;
6886         die "mirroring error: $err\n";
6887     }
6888
6889     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6890 }
6891
6892 sub qemu_drive_mirror_monitor {
6893     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6894
6895     eval {
6896         my $err_complete = 0;
6897
6898         while (1) {
6899             die "storage migration timed out\n" if $err_complete > 300;
6900
6901             my $stats = mon_cmd($vmid, "query-block-jobs");
6902
6903             my $running_mirror_jobs = {};
6904             foreach my $stat (@$stats) {
6905                 next if $stat->{type} ne 'mirror';
6906                 $running_mirror_jobs->{$stat->{device}} = $stat;
6907             }
6908
6909             my $readycounter = 0;
6910
6911             foreach my $job (keys %$jobs) {
6912
6913                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6914                     print "$job : finished\n";
6915                     delete $jobs->{$job};
6916                     next;
6917                 }
6918
6919                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6920
6921                 my $busy = $running_mirror_jobs->{$job}->{busy};
6922                 my $ready = $running_mirror_jobs->{$job}->{ready};
6923                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6924                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6925                     my $remaining = $total - $transferred;
6926                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6927
6928                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6929                 }
6930
6931                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6932             }
6933
6934             last if scalar(keys %$jobs) == 0;
6935
6936             if ($readycounter == scalar(keys %$jobs)) {
6937                 print "all mirroring jobs are ready \n";
6938                 last if $skipcomplete; #do the complete later
6939
6940                 if ($vmiddst && $vmiddst != $vmid) {
6941                     my $agent_running = $qga && qga_check_running($vmid);
6942                     if ($agent_running) {
6943                         print "freeze filesystem\n";
6944                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6945                     } else {
6946                         print "suspend vm\n";
6947                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6948                     }
6949
6950                     # if we clone a disk for a new target vm, we don't switch the disk
6951                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6952
6953                     if ($agent_running) {
6954                         print "unfreeze filesystem\n";
6955                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6956                     } else {
6957                         print "resume vm\n";
6958                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6959                     }
6960
6961                     last;
6962                 } else {
6963
6964                     foreach my $job (keys %$jobs) {
6965                         # try to switch the disk if source and destination are on the same guest
6966                         print "$job: Completing block job...\n";
6967
6968                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6969                         if ($@ =~ m/cannot be completed/) {
6970                             print "$job: Block job cannot be completed, try again.\n";
6971                             $err_complete++;
6972                         }else {
6973                             print "$job: Completed successfully.\n";
6974                             $jobs->{$job}->{complete} = 1;
6975                         }
6976                     }
6977                 }
6978             }
6979             sleep 1;
6980         }
6981     };
6982     my $err = $@;
6983
6984     if ($err) {
6985         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6986         die "mirroring error: $err";
6987     }
6988
6989 }
6990
6991 sub qemu_blockjobs_cancel {
6992     my ($vmid, $jobs) = @_;
6993
6994     foreach my $job (keys %$jobs) {
6995         print "$job: Cancelling block job\n";
6996         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6997         $jobs->{$job}->{cancel} = 1;
6998     }
6999
7000     while (1) {
7001         my $stats = mon_cmd($vmid, "query-block-jobs");
7002
7003         my $running_jobs = {};
7004         foreach my $stat (@$stats) {
7005             $running_jobs->{$stat->{device}} = $stat;
7006         }
7007
7008         foreach my $job (keys %$jobs) {
7009
7010             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7011                 print "$job: Done.\n";
7012                 delete $jobs->{$job};
7013             }
7014         }
7015
7016         last if scalar(keys %$jobs) == 0;
7017
7018         sleep 1;
7019     }
7020 }
7021
7022 sub clone_disk {
7023     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7024         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7025
7026     my $newvolid;
7027
7028     if (!$full) {
7029         print "create linked clone of drive $drivename ($drive->{file})\n";
7030         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7031         push @$newvollist, $newvolid;
7032     } else {
7033
7034         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7035         $storeid = $storage if $storage;
7036
7037         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7038         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7039
7040         print "create full clone of drive $drivename ($drive->{file})\n";
7041         my $name = undef;
7042         if (drive_is_cloudinit($drive)) {
7043             $name = "vm-$newvmid-cloudinit";
7044             $name .= ".$dst_format" if $dst_format ne 'raw';
7045             $snapname = undef;
7046             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7047         }
7048         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7049         push @$newvollist, $newvolid;
7050
7051         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7052
7053         if (drive_is_cloudinit($drive)) {
7054             goto no_data_clone;
7055         }
7056
7057         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7058         if (!$running || $snapname) {
7059             # TODO: handle bwlimits
7060             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7061         } else {
7062
7063             my $kvmver = get_running_qemu_version ($vmid);
7064             if (!min_version($kvmver, 2, 7)) {
7065                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7066                     if $drive->{iothread};
7067             }
7068
7069             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7070         }
7071     }
7072
7073 no_data_clone:
7074     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7075
7076     my $disk = $drive;
7077     $disk->{format} = undef;
7078     $disk->{file} = $newvolid;
7079     $disk->{size} = $size;
7080
7081     return $disk;
7082 }
7083
7084 sub get_running_qemu_version {
7085     my ($vmid) = @_;
7086     my $res = mon_cmd($vmid, "query-version");
7087     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7088 }
7089
7090 sub qemu_use_old_bios_files {
7091     my ($machine_type) = @_;
7092
7093     return if !$machine_type;
7094
7095     my $use_old_bios_files = undef;
7096
7097     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7098         $machine_type = $1;
7099         $use_old_bios_files = 1;
7100     } else {
7101         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7102         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7103         # load new efi bios files on migration. So this hack is required to allow
7104         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7105         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7106         $use_old_bios_files = !min_version($version, 2, 4);
7107     }
7108
7109     return ($use_old_bios_files, $machine_type);
7110 }
7111
7112 sub create_efidisk($$$$$) {
7113     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7114
7115     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7116     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7117
7118     my $vars_size_b = -s $ovmf_vars;
7119     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7120     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7121     PVE::Storage::activate_volumes($storecfg, [$volid]);
7122
7123     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7124     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7125
7126     return ($volid, $size/1024);
7127 }
7128
7129 sub vm_iothreads_list {
7130     my ($vmid) = @_;
7131
7132     my $res = mon_cmd($vmid, 'query-iothreads');
7133
7134     my $iothreads = {};
7135     foreach my $iothread (@$res) {
7136         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7137     }
7138
7139     return $iothreads;
7140 }
7141
7142 sub scsihw_infos {
7143     my ($conf, $drive) = @_;
7144
7145     my $maxdev = 0;
7146
7147     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7148         $maxdev = 7;
7149     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7150         $maxdev = 1;
7151     } else {
7152         $maxdev = 256;
7153     }
7154
7155     my $controller = int($drive->{index} / $maxdev);
7156     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7157
7158     return ($maxdev, $controller, $controller_prefix);
7159 }
7160
7161 sub windows_version {
7162     my ($ostype) = @_;
7163
7164     return 0 if !$ostype;
7165
7166     my $winversion = 0;
7167
7168     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7169         $winversion = 5;
7170     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7171         $winversion = 6;
7172     } elsif ($ostype =~ m/^win(\d+)$/) {
7173         $winversion = $1;
7174     }
7175
7176     return $winversion;
7177 }
7178
7179 sub resolve_dst_disk_format {
7180         my ($storecfg, $storeid, $src_volname, $format) = @_;
7181         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7182
7183         if (!$format) {
7184             # if no target format is specified, use the source disk format as hint
7185             if ($src_volname) {
7186                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7187                 $format = qemu_img_format($scfg, $src_volname);
7188             } else {
7189                 return $defFormat;
7190             }
7191         }
7192
7193         # test if requested format is supported - else use default
7194         my $supported = grep { $_ eq $format } @$validFormats;
7195         $format = $defFormat if !$supported;
7196         return $format;
7197 }
7198
7199 sub resolve_first_disk {
7200     my $conf = shift;
7201     my @disks = PVE::QemuServer::valid_drive_names();
7202     my $firstdisk;
7203     foreach my $ds (reverse @disks) {
7204         next if !$conf->{$ds};
7205         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7206         next if PVE::QemuServer::drive_is_cdrom($disk);
7207         $firstdisk = $ds;
7208     }
7209     return $firstdisk;
7210 }
7211
7212 # NOTE: if this logic changes, please update docs & possibly gui logic
7213 sub find_vmstate_storage {
7214     my ($conf, $storecfg) = @_;
7215
7216     # first, return storage from conf if set
7217     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7218
7219     my ($target, $shared, $local);
7220
7221     foreach_storage_used_by_vm($conf, sub {
7222         my ($sid) = @_;
7223         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7224         my $dst = $scfg->{shared} ? \$shared : \$local;
7225         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7226     });
7227
7228     # second, use shared storage where VM has at least one disk
7229     # third, use local storage where VM has at least one disk
7230     # fall back to local storage
7231     $target = $shared // $local // 'local';
7232
7233     return $target;
7234 }
7235
7236 sub generate_uuid {
7237     my ($uuid, $uuid_str);
7238     UUID::generate($uuid);
7239     UUID::unparse($uuid, $uuid_str);
7240     return $uuid_str;
7241 }
7242
7243 sub generate_smbios1_uuid {
7244     return "uuid=".generate_uuid();
7245 }
7246
7247 sub nbd_stop {
7248     my ($vmid) = @_;
7249
7250     mon_cmd($vmid, 'nbd-server-stop');
7251 }
7252
7253 sub create_reboot_request {
7254     my ($vmid) = @_;
7255     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7256         or die "failed to create reboot trigger file: $!\n";
7257     close($fh);
7258 }
7259
7260 sub clear_reboot_request {
7261     my ($vmid) = @_;
7262     my $path = "/run/qemu-server/$vmid.reboot";
7263     my $res = 0;
7264
7265     $res = unlink($path);
7266     die "could not remove reboot request for $vmid: $!"
7267         if !$res && $! != POSIX::ENOENT;
7268
7269     return $res;
7270 }
7271
7272 # bash completion helper
7273
7274 sub complete_backup_archives {
7275     my ($cmdname, $pname, $cvalue) = @_;
7276
7277     my $cfg = PVE::Storage::config();
7278
7279     my $storeid;
7280
7281     if ($cvalue =~ m/^([^:]+):/) {
7282         $storeid = $1;
7283     }
7284
7285     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7286
7287     my $res = [];
7288     foreach my $id (keys %$data) {
7289         foreach my $item (@{$data->{$id}}) {
7290             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7291             push @$res, $item->{volid} if defined($item->{volid});
7292         }
7293     }
7294
7295     return $res;
7296 }
7297
7298 my $complete_vmid_full = sub {
7299     my ($running) = @_;
7300
7301     my $idlist = vmstatus();
7302
7303     my $res = [];
7304
7305     foreach my $id (keys %$idlist) {
7306         my $d = $idlist->{$id};
7307         if (defined($running)) {
7308             next if $d->{template};
7309             next if $running && $d->{status} ne 'running';
7310             next if !$running && $d->{status} eq 'running';
7311         }
7312         push @$res, $id;
7313
7314     }
7315     return $res;
7316 };
7317
7318 sub complete_vmid {
7319     return &$complete_vmid_full();
7320 }
7321
7322 sub complete_vmid_stopped {
7323     return &$complete_vmid_full(0);
7324 }
7325
7326 sub complete_vmid_running {
7327     return &$complete_vmid_full(1);
7328 }
7329
7330 sub complete_storage {
7331
7332     my $cfg = PVE::Storage::config();
7333     my $ids = $cfg->{ids};
7334
7335     my $res = [];
7336     foreach my $sid (keys %$ids) {
7337         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7338         next if !$ids->{$sid}->{content}->{images};
7339         push @$res, $sid;
7340     }
7341
7342     return $res;
7343 }
7344
7345 sub complete_migration_storage {
7346     my ($cmd, $param, $current_value, $all_args) = @_;
7347
7348     my $targetnode = @$all_args[1];
7349
7350     my $cfg = PVE::Storage::config();
7351     my $ids = $cfg->{ids};
7352
7353     my $res = [];
7354     foreach my $sid (keys %$ids) {
7355         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7356         next if !$ids->{$sid}->{content}->{images};
7357         push @$res, $sid;
7358     }
7359
7360     return $res;
7361 }
7362
7363 1;