PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Machine;
  48 use PVE::QemuServer::Memory;
  49 use PVE::QemuServer::Monitor qw(mon_cmd);
  50 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  51 use PVE::QemuServer::USB qw(parse_usb_device);
  52
  53 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  54 my $OVMF = {
  55     x86_64 => [
  56         "$EDK2_FW_BASE/OVMF_CODE.fd",
  57         "$EDK2_FW_BASE/OVMF_VARS.fd"
  58     ],
  59     aarch64 => [
  60         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  61         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  62     ],
  63 };
  64
  65 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  66
  67 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  68
  69 # Note about locking: we use flock on the config file protect
  70 # against concurent actions.
  71 # Aditionaly, we have a 'lock' setting in the config file. This
  72 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  73 # allowed when such lock is set. But you can ignore this kind of
  74 # lock with the --skiplock flag.
  75
  76 cfs_register_file('/qemu-server/',
  77                   \&parse_vm_config,
  78                   \&write_vm_config);
  79
  80 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  81     description => "Some command save/restore state from this location.",
  82     type => 'string',
  83     maxLength => 128,
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  88     type => 'string',
  89     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  90     description => "The drive's backing file's data format.",
  91     optional => 1,
  92 });
  93
  94 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  95         description => "Specifies the Qemu machine type.",
  96         type => 'string',
  97         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  98         maxLength => 40,
  99         optional => 1,
 100 });
 101
 102 #no warnings 'redefine';
 103
 104 sub cgroups_write {
 105    my ($controller, $vmid, $option, $value) = @_;
 106
 107    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 108    PVE::ProcFSTools::write_proc_entry($path, $value);
 109
 110 }
 111
 112 my $nodename_cache;
 113 sub nodename {
 114     $nodename_cache //= PVE::INotify::nodename();
 115     return $nodename_cache;
 116 }
 117
 118 my $watchdog_fmt = {
 119     model => {
 120         default_key => 1,
 121         type => 'string',
 122         enum => [qw(i6300esb ib700)],
 123         description => "Watchdog type to emulate.",
 124         default => 'i6300esb',
 125         optional => 1,
 126     },
 127     action => {
 128         type => 'string',
 129         enum => [qw(reset shutdown poweroff pause debug none)],
 130         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 131         optional => 1,
 132     },
 133 };
 134 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 135
 136 my $agent_fmt = {
 137     enabled => {
 138         description => "Enable/disable Qemu GuestAgent.",
 139         type => 'boolean',
 140         default => 0,
 141         default_key => 1,
 142     },
 143     fstrim_cloned_disks => {
 144         description => "Run fstrim after cloning/moving a disk.",
 145         type => 'boolean',
 146         optional => 1,
 147         default => 0
 148     },
 149     type => {
 150         description => "Select the agent type",
 151         type => 'string',
 152         default => 'virtio',
 153         optional => 1,
 154         enum => [qw(virtio isa)],
 155     },
 156 };
 157
 158 my $vga_fmt = {
 159     type => {
 160         description => "Select the VGA type.",
 161         type => 'string',
 162         default => 'std',
 163         optional => 1,
 164         default_key => 1,
 165         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 166     },
 167     memory => {
 168         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 169         type => 'integer',
 170         optional => 1,
 171         minimum => 4,
 172         maximum => 512,
 173     },
 174 };
 175
 176 my $ivshmem_fmt = {
 177     size => {
 178         type => 'integer',
 179         minimum => 1,
 180         description => "The size of the file in MB.",
 181     },
 182     name => {
 183         type => 'string',
 184         pattern => '[a-zA-Z0-9\-]+',
 185         optional => 1,
 186         format_description => 'string',
 187         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 188     },
 189 };
 190
 191 my $audio_fmt = {
 192     device => {
 193         type => 'string',
 194         enum => [qw(ich9-intel-hda intel-hda AC97)],
 195         description =>  "Configure an audio device."
 196     },
 197     driver =>  {
 198         type => 'string',
 199         enum => ['spice'],
 200         default => 'spice',
 201         optional => 1,
 202         description => "Driver backend for the audio device."
 203     },
 204 };
 205
 206 my $spice_enhancements_fmt = {
 207     foldersharing => {
 208         type => 'boolean',
 209         optional => 1,
 210         default => '0',
 211         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 212     },
 213     videostreaming =>  {
 214         type => 'string',
 215         enum => ['off', 'all', 'filter'],
 216         default => 'off',
 217         optional => 1,
 218         description => "Enable video streaming. Uses compression for detected video streams."
 219     },
 220 };
 221
 222 my $confdesc = {
 223     onboot => {
 224         optional => 1,
 225         type => 'boolean',
 226         description => "Specifies whether a VM will be started during system bootup.",
 227         default => 0,
 228     },
 229     autostart => {
 230         optional => 1,
 231         type => 'boolean',
 232         description => "Automatic restart after crash (currently ignored).",
 233         default => 0,
 234     },
 235     hotplug => {
 236         optional => 1,
 237         type => 'string', format => 'pve-hotplug-features',
 238         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 239         default => 'network,disk,usb',
 240     },
 241     reboot => {
 242         optional => 1,
 243         type => 'boolean',
 244         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 245         default => 1,
 246     },
 247     lock => {
 248         optional => 1,
 249         type => 'string',
 250         description => "Lock/unlock the VM.",
 251         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 252     },
 253     cpulimit => {
 254         optional => 1,
 255         type => 'number',
 256         description => "Limit of CPU usage.",
 257         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 258         minimum => 0,
 259         maximum => 128,
 260         default => 0,
 261     },
 262     cpuunits => {
 263         optional => 1,
 264         type => 'integer',
 265         description => "CPU weight for a VM.",
 266         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 267         minimum => 2,
 268         maximum => 262144,
 269         default => 1024,
 270     },
 271     memory => {
 272         optional => 1,
 273         type => 'integer',
 274         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 275         minimum => 16,
 276         default => 512,
 277     },
 278     balloon => {
 279         optional => 1,
 280         type => 'integer',
 281         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 282         minimum => 0,
 283     },
 284     shares => {
 285         optional => 1,
 286         type => 'integer',
 287         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 288         minimum => 0,
 289         maximum => 50000,
 290         default => 1000,
 291     },
 292     keyboard => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 296                        "It should not be necessary to set it.",
 297         enum => PVE::Tools::kvmkeymaplist(),
 298         default => undef,
 299     },
 300     name => {
 301         optional => 1,
 302         type => 'string', format => 'dns-name',
 303         description => "Set a name for the VM. Only used on the configuration web interface.",
 304     },
 305     scsihw => {
 306         optional => 1,
 307         type => 'string',
 308         description => "SCSI controller model",
 309         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 310         default => 'lsi',
 311     },
 312     description => {
 313         optional => 1,
 314         type => 'string',
 315         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 316     },
 317     ostype => {
 318         optional => 1,
 319         type => 'string',
 320         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 321         description => "Specify guest operating system.",
 322         verbose_description => <<EODESC,
 323 Specify guest operating system. This is used to enable special
 324 optimization/features for specific operating systems:
 325
 326 [horizontal]
 327 other;; unspecified OS
 328 wxp;; Microsoft Windows XP
 329 w2k;; Microsoft Windows 2000
 330 w2k3;; Microsoft Windows 2003
 331 w2k8;; Microsoft Windows 2008
 332 wvista;; Microsoft Windows Vista
 333 win7;; Microsoft Windows 7
 334 win8;; Microsoft Windows 8/2012/2012r2
 335 win10;; Microsoft Windows 10/2016
 336 l24;; Linux 2.4 Kernel
 337 l26;; Linux 2.6 - 5.X Kernel
 338 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 339 EODESC
 340     },
 341     boot => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 345         pattern => '[acdn]{1,4}',
 346         default => 'cdn',
 347     },
 348     bootdisk => {
 349         optional => 1,
 350         type => 'string', format => 'pve-qm-bootdisk',
 351         description => "Enable booting from specified disk.",
 352         pattern => '(ide|sata|scsi|virtio)\d+',
 353     },
 354     smp => {
 355         optional => 1,
 356         type => 'integer',
 357         description => "The number of CPUs. Please use option -sockets instead.",
 358         minimum => 1,
 359         default => 1,
 360     },
 361     sockets => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "The number of CPU sockets.",
 365         minimum => 1,
 366         default => 1,
 367     },
 368     cores => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "The number of cores per socket.",
 372         minimum => 1,
 373         default => 1,
 374     },
 375     numa => {
 376         optional => 1,
 377         type => 'boolean',
 378         description => "Enable/disable NUMA.",
 379         default => 0,
 380     },
 381     hugepages => {
 382         optional => 1,
 383         type => 'string',
 384         description => "Enable/disable hugepages memory.",
 385         enum => [qw(any 2 1024)],
 386     },
 387     vcpus => {
 388         optional => 1,
 389         type => 'integer',
 390         description => "Number of hotplugged vcpus.",
 391         minimum => 1,
 392         default => 0,
 393     },
 394     acpi => {
 395         optional => 1,
 396         type => 'boolean',
 397         description => "Enable/disable ACPI.",
 398         default => 1,
 399     },
 400     agent => {
 401         optional => 1,
 402         description => "Enable/disable Qemu GuestAgent and its properties.",
 403         type => 'string',
 404         format => $agent_fmt,
 405     },
 406     kvm => {
 407         optional => 1,
 408         type => 'boolean',
 409         description => "Enable/disable KVM hardware virtualization.",
 410         default => 1,
 411     },
 412     tdf => {
 413         optional => 1,
 414         type => 'boolean',
 415         description => "Enable/disable time drift fix.",
 416         default => 0,
 417     },
 418     localtime => {
 419         optional => 1,
 420         type => 'boolean',
 421         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 422     },
 423     freeze => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 427     },
 428     vga => {
 429         optional => 1,
 430         type => 'string', format => $vga_fmt,
 431         description => "Configure the VGA hardware.",
 432         verbose_description => "Configure the VGA Hardware. If you want to use ".
 433             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 434             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 435             "is 'std' for all OS types besides some Windows versions (XP and " .
 436             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 437             "display server. For win* OS you can select how many independent " .
 438             "displays you want, Linux guests can add displays them self.\n".
 439             "You can also run without any graphic card, using a serial device as terminal.",
 440     },
 441     watchdog => {
 442         optional => 1,
 443         type => 'string', format => 'pve-qm-watchdog',
 444         description => "Create a virtual hardware watchdog device.",
 445         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 446             " (by a guest action), the watchdog must be periodically polled " .
 447             "by an agent inside the guest or else the watchdog will reset " .
 448             "the guest (or execute the respective action specified)",
 449     },
 450     startdate => {
 451         optional => 1,
 452         type => 'string',
 453         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 454         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 455         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 456         default => 'now',
 457     },
 458     startup =>  get_standard_option('pve-startup-order'),
 459     template => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Enable/disable Template.",
 463         default => 0,
 464     },
 465     args => {
 466         optional => 1,
 467         type => 'string',
 468         description => "Arbitrary arguments passed to kvm.",
 469         verbose_description => <<EODESCR,
 470 Arbitrary arguments passed to kvm, for example:
 471
 472 args: -no-reboot -no-hpet
 473
 474 NOTE: this option is for experts only.
 475 EODESCR
 476     },
 477     tablet => {
 478         optional => 1,
 479         type => 'boolean',
 480         default => 1,
 481         description => "Enable/disable the USB tablet device.",
 482         verbose_description => "Enable/disable the USB tablet device. This device is " .
 483             "usually needed to allow absolute mouse positioning with VNC. " .
 484             "Else the mouse runs out of sync with normal VNC clients. " .
 485             "If you're running lots of console-only guests on one host, " .
 486             "you may consider disabling this to save some context switches. " .
 487             "This is turned off by default if you use spice (-vga=qxl).",
 488     },
 489     migrate_speed => {
 490         optional => 1,
 491         type => 'integer',
 492         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 493         minimum => 0,
 494         default => 0,
 495     },
 496     migrate_downtime => {
 497         optional => 1,
 498         type => 'number',
 499         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 500         minimum => 0,
 501         default => 0.1,
 502     },
 503     cdrom => {
 504         optional => 1,
 505         type => 'string', format => 'pve-qm-ide',
 506         typetext => '<volume>',
 507         description => "This is an alias for option -ide2",
 508     },
 509     cpu => {
 510         optional => 1,
 511         description => "Emulated CPU type.",
 512         type => 'string',
 513         format => $PVE::QemuServer::CPUConfig::cpu_fmt,
 514     },
 515     parent => get_standard_option('pve-snapshot-name', {
 516         optional => 1,
 517         description => "Parent snapshot name. This is used internally, and should not be modified.",
 518     }),
 519     snaptime => {
 520         optional => 1,
 521         description => "Timestamp for snapshots.",
 522         type => 'integer',
 523         minimum => 0,
 524     },
 525     vmstate => {
 526         optional => 1,
 527         type => 'string', format => 'pve-volume-id',
 528         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 529     },
 530     vmstatestorage => get_standard_option('pve-storage-id', {
 531         description => "Default storage for VM state volumes/files.",
 532         optional => 1,
 533     }),
 534     runningmachine => get_standard_option('pve-qemu-machine', {
 535         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 536     }),
 537     machine => get_standard_option('pve-qemu-machine'),
 538     arch => {
 539         description => "Virtual processor architecture. Defaults to the host.",
 540         optional => 1,
 541         type => 'string',
 542         enum => [qw(x86_64 aarch64)],
 543     },
 544     smbios1 => {
 545         description => "Specify SMBIOS type 1 fields.",
 546         type => 'string', format => 'pve-qm-smbios1',
 547         maxLength => 512,
 548         optional => 1,
 549     },
 550     protection => {
 551         optional => 1,
 552         type => 'boolean',
 553         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 554         default => 0,
 555     },
 556     bios => {
 557         optional => 1,
 558         type => 'string',
 559         enum => [ qw(seabios ovmf) ],
 560         description => "Select BIOS implementation.",
 561         default => 'seabios',
 562     },
 563     vmgenid => {
 564         type => 'string',
 565         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 566         format_description => 'UUID',
 567         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 568         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 569             " 128-bit integer value identifier to the guest OS. This allows to".
 570             " notify the guest operating system when the virtual machine is".
 571             " executed with a different configuration (e.g. snapshot execution".
 572             " or creation from a template). The guest operating system notices".
 573             " the change, and is then able to react as appropriate by marking".
 574             " its copies of distributed databases as dirty, re-initializing its".
 575             " random number generator, etc.\n".
 576             "Note that auto-creation only works when done throug API/CLI create".
 577             " or update methods, but not when manually editing the config file.",
 578         default => "1 (autogenerated)",
 579         optional => 1,
 580     },
 581     hookscript => {
 582         type => 'string',
 583         format => 'pve-volume-id',
 584         optional => 1,
 585         description => "Script that will be executed during various steps in the vms lifetime.",
 586     },
 587     ivshmem => {
 588         type => 'string',
 589         format => $ivshmem_fmt,
 590         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 591         optional => 1,
 592     },
 593     audio0 => {
 594         type => 'string',
 595         format => $audio_fmt,
 596         description => "Configure a audio device, useful in combination with QXL/Spice.",
 597         optional => 1
 598     },
 599     spice_enhancements => {
 600         type => 'string',
 601         format => $spice_enhancements_fmt,
 602         description => "Configure additional enhancements for SPICE.",
 603         optional => 1
 604     },
 605     tags => {
 606         type => 'string', format => 'pve-tag-list',
 607         description => 'Tags of the VM. This is only meta information.',
 608         optional => 1,
 609     },
 610 };
 611
 612 my $cicustom_fmt = {
 613     meta => {
 614         type => 'string',
 615         optional => 1,
 616         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 617         format => 'pve-volume-id',
 618         format_description => 'volume',
 619     },
 620     network => {
 621         type => 'string',
 622         optional => 1,
 623         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 624         format => 'pve-volume-id',
 625         format_description => 'volume',
 626     },
 627     user => {
 628         type => 'string',
 629         optional => 1,
 630         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 631         format => 'pve-volume-id',
 632         format_description => 'volume',
 633     },
 634 };
 635 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 636
 637 my $confdesc_cloudinit = {
 638     citype => {
 639         optional => 1,
 640         type => 'string',
 641         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 642         enum => ['configdrive2', 'nocloud'],
 643     },
 644     ciuser => {
 645         optional => 1,
 646         type => 'string',
 647         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 648     },
 649     cipassword => {
 650         optional => 1,
 651         type => 'string',
 652         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 653     },
 654     cicustom => {
 655         optional => 1,
 656         type => 'string',
 657         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 658         format => 'pve-qm-cicustom',
 659     },
 660     searchdomain => {
 661         optional => 1,
 662         type => 'string',
 663         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 664     },
 665     nameserver => {
 666         optional => 1,
 667         type => 'string', format => 'address-list',
 668         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 669     },
 670     sshkeys => {
 671         optional => 1,
 672         type => 'string',
 673         format => 'urlencoded',
 674         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 675     },
 676 };
 677
 678 # what about other qemu settings ?
 679 #cpu => 'string',
 680 #machine => 'string',
 681 #fda => 'file',
 682 #fdb => 'file',
 683 #mtdblock => 'file',
 684 #sd => 'file',
 685 #pflash => 'file',
 686 #snapshot => 'bool',
 687 #bootp => 'file',
 688 ##tftp => 'dir',
 689 ##smb => 'dir',
 690 #kernel => 'file',
 691 #append => 'string',
 692 #initrd => 'file',
 693 ##soundhw => 'string',
 694
 695 while (my ($k, $v) = each %$confdesc) {
 696     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 697 }
 698
 699 my $MAX_IDE_DISKS = 4;
 700 my $MAX_SCSI_DISKS = 31;
 701 my $MAX_VIRTIO_DISKS = 16;
 702 my $MAX_SATA_DISKS = 6;
 703 my $MAX_USB_DEVICES = 5;
 704 my $MAX_NETS = 32;
 705 my $MAX_UNUSED_DISKS = 256;
 706 my $MAX_HOSTPCI_DEVICES = 16;
 707 my $MAX_SERIAL_PORTS = 4;
 708 my $MAX_PARALLEL_PORTS = 3;
 709 my $MAX_NUMA = 8;
 710
 711 my $numa_fmt = {
 712     cpus => {
 713         type => "string",
 714         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 715         description => "CPUs accessing this NUMA node.",
 716         format_description => "id[-id];...",
 717     },
 718     memory => {
 719         type => "number",
 720         description => "Amount of memory this NUMA node provides.",
 721         optional => 1,
 722     },
 723     hostnodes => {
 724         type => "string",
 725         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 726         description => "Host NUMA nodes to use.",
 727         format_description => "id[-id];...",
 728         optional => 1,
 729     },
 730     policy => {
 731         type => 'string',
 732         enum => [qw(preferred bind interleave)],
 733         description => "NUMA allocation policy.",
 734         optional => 1,
 735     },
 736 };
 737 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 738 my $numadesc = {
 739     optional => 1,
 740     type => 'string', format => $numa_fmt,
 741     description => "NUMA topology.",
 742 };
 743 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 744
 745 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 746     $confdesc->{"numa$i"} = $numadesc;
 747 }
 748
 749 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 750                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 751                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 752 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 753
 754 my $net_fmt_bridge_descr = <<__EOD__;
 755 Bridge to attach the network device to. The Proxmox VE standard bridge
 756 is called 'vmbr0'.
 757
 758 If you do not specify a bridge, we create a kvm user (NATed) network
 759 device, which provides DHCP and DNS services. The following addresses
 760 are used:
 761
 762  10.0.2.2   Gateway
 763  10.0.2.3   DNS Server
 764  10.0.2.4   SMB Server
 765
 766 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 767 __EOD__
 768
 769 my $net_fmt = {
 770     macaddr  => get_standard_option('mac-addr', {
 771         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 772     }),
 773     model => {
 774         type => 'string',
 775         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 776         enum => $nic_model_list,
 777         default_key => 1,
 778     },
 779     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 780     bridge => {
 781         type => 'string',
 782         description => $net_fmt_bridge_descr,
 783         format_description => 'bridge',
 784         optional => 1,
 785     },
 786     queues => {
 787         type => 'integer',
 788         minimum => 0, maximum => 16,
 789         description => 'Number of packet queues to be used on the device.',
 790         optional => 1,
 791     },
 792     rate => {
 793         type => 'number',
 794         minimum => 0,
 795         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 796         optional => 1,
 797     },
 798     tag => {
 799         type => 'integer',
 800         minimum => 1, maximum => 4094,
 801         description => 'VLAN tag to apply to packets on this interface.',
 802         optional => 1,
 803     },
 804     trunks => {
 805         type => 'string',
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => 'VLAN trunks to pass through this interface.',
 808         format_description => 'vlanid[;vlanid...]',
 809         optional => 1,
 810     },
 811     firewall => {
 812         type => 'boolean',
 813         description => 'Whether this interface should be protected by the firewall.',
 814         optional => 1,
 815     },
 816     link_down => {
 817         type => 'boolean',
 818         description => 'Whether this interface should be disconnected (like pulling the plug).',
 819         optional => 1,
 820     },
 821 };
 822
 823 my $netdesc = {
 824     optional => 1,
 825     type => 'string', format => $net_fmt,
 826     description => "Specify network devices.",
 827 };
 828
 829 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 830
 831 my $ipconfig_fmt = {
 832     ip => {
 833         type => 'string',
 834         format => 'pve-ipv4-config',
 835         format_description => 'IPv4Format/CIDR',
 836         description => 'IPv4 address in CIDR format.',
 837         optional => 1,
 838         default => 'dhcp',
 839     },
 840     gw => {
 841         type => 'string',
 842         format => 'ipv4',
 843         format_description => 'GatewayIPv4',
 844         description => 'Default gateway for IPv4 traffic.',
 845         optional => 1,
 846         requires => 'ip',
 847     },
 848     ip6 => {
 849         type => 'string',
 850         format => 'pve-ipv6-config',
 851         format_description => 'IPv6Format/CIDR',
 852         description => 'IPv6 address in CIDR format.',
 853         optional => 1,
 854         default => 'dhcp',
 855     },
 856     gw6 => {
 857         type => 'string',
 858         format => 'ipv6',
 859         format_description => 'GatewayIPv6',
 860         description => 'Default gateway for IPv6 traffic.',
 861         optional => 1,
 862         requires => 'ip6',
 863     },
 864 };
 865 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 866 my $ipconfigdesc = {
 867     optional => 1,
 868     type => 'string', format => 'pve-qm-ipconfig',
 869     description => <<'EODESCR',
 870 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 871
 872 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 873
 874 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 875 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 876
 877 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 878 EODESCR
 879 };
 880 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 881
 882 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 883     $confdesc->{"net$i"} = $netdesc;
 884     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 885 }
 886
 887 foreach my $key (keys %$confdesc_cloudinit) {
 888     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 889 }
 890
 891 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 892 sub verify_volume_id_or_qm_path {
 893     my ($volid, $noerr) = @_;
 894
 895     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 896         return $volid;
 897     }
 898
 899     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 900     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 901     if ($@) {
 902         return undef if $noerr;
 903         die $@;
 904     }
 905     return $volid;
 906 }
 907
 908 my $drivename_hash;
 909
 910 my %drivedesc_base = (
 911     volume => { alias => 'file' },
 912     file => {
 913         type => 'string',
 914         format => 'pve-volume-id-or-qm-path',
 915         default_key => 1,
 916         format_description => 'volume',
 917         description => "The drive's backing volume.",
 918     },
 919     media => {
 920         type => 'string',
 921         enum => [qw(cdrom disk)],
 922         description => "The drive's media type.",
 923         default => 'disk',
 924         optional => 1
 925     },
 926     cyls => {
 927         type => 'integer',
 928         description => "Force the drive's physical geometry to have a specific cylinder count.",
 929         optional => 1
 930     },
 931     heads => {
 932         type => 'integer',
 933         description => "Force the drive's physical geometry to have a specific head count.",
 934         optional => 1
 935     },
 936     secs => {
 937         type => 'integer',
 938         description => "Force the drive's physical geometry to have a specific sector count.",
 939         optional => 1
 940     },
 941     trans => {
 942         type => 'string',
 943         enum => [qw(none lba auto)],
 944         description => "Force disk geometry bios translation mode.",
 945         optional => 1,
 946     },
 947     snapshot => {
 948         type => 'boolean',
 949         description => "Controls qemu's snapshot mode feature."
 950             . " If activated, changes made to the disk are temporary and will"
 951             . " be discarded when the VM is shutdown.",
 952         optional => 1,
 953     },
 954     cache => {
 955         type => 'string',
 956         enum => [qw(none writethrough writeback unsafe directsync)],
 957         description => "The drive's cache mode",
 958         optional => 1,
 959     },
 960     format => get_standard_option('pve-qm-image-format'),
 961     size => {
 962         type => 'string',
 963         format => 'disk-size',
 964         format_description => 'DiskSize',
 965         description => "Disk size. This is purely informational and has no effect.",
 966         optional => 1,
 967     },
 968     backup => {
 969         type => 'boolean',
 970         description => "Whether the drive should be included when making backups.",
 971         optional => 1,
 972     },
 973     replicate => {
 974         type => 'boolean',
 975         description => 'Whether the drive should considered for replication jobs.',
 976         optional => 1,
 977         default => 1,
 978     },
 979     rerror => {
 980         type => 'string',
 981         enum => [qw(ignore report stop)],
 982         description => 'Read error action.',
 983         optional => 1,
 984     },
 985     werror => {
 986         type => 'string',
 987         enum => [qw(enospc ignore report stop)],
 988         description => 'Write error action.',
 989         optional => 1,
 990     },
 991     aio => {
 992         type => 'string',
 993         enum => [qw(native threads)],
 994         description => 'AIO type to use.',
 995         optional => 1,
 996     },
 997     discard => {
 998         type => 'string',
 999         enum => [qw(ignore on)],
1000         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1001         optional => 1,
1002     },
1003     detect_zeroes => {
1004         type => 'boolean',
1005         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1006         optional => 1,
1007     },
1008     serial => {
1009         type => 'string',
1010         format => 'urlencoded',
1011         format_description => 'serial',
1012         maxLength => 20*3, # *3 since it's %xx url enoded
1013         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1014         optional => 1,
1015     },
1016     shared => {
1017         type => 'boolean',
1018         description => 'Mark this locally-managed volume as available on all nodes',
1019         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1020         optional => 1,
1021         default => 0,
1022     }
1023 );
1024
1025 my %iothread_fmt = ( iothread => {
1026         type => 'boolean',
1027         description => "Whether to use iothreads for this drive",
1028         optional => 1,
1029 });
1030
1031 my %model_fmt = (
1032     model => {
1033         type => 'string',
1034         format => 'urlencoded',
1035         format_description => 'model',
1036         maxLength => 40*3, # *3 since it's %xx url enoded
1037         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1038         optional => 1,
1039     },
1040 );
1041
1042 my %queues_fmt = (
1043     queues => {
1044         type => 'integer',
1045         description => "Number of queues.",
1046         minimum => 2,
1047         optional => 1
1048     }
1049 );
1050
1051 my %scsiblock_fmt = (
1052     scsiblock => {
1053         type => 'boolean',
1054         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1055         optional => 1,
1056         default => 0,
1057     },
1058 );
1059
1060 my %ssd_fmt = (
1061     ssd => {
1062         type => 'boolean',
1063         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1064         optional => 1,
1065     },
1066 );
1067
1068 my %wwn_fmt = (
1069     wwn => {
1070         type => 'string',
1071         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1072         format_description => 'wwn',
1073         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1074         optional => 1,
1075     },
1076 );
1077
1078 my $add_throttle_desc = sub {
1079     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1080     my $d = {
1081         type => $type,
1082         format_description => $unit,
1083         description => "Maximum $what in $longunit.",
1084         optional => 1,
1085     };
1086     $d->{minimum} = $minimum if defined($minimum);
1087     $drivedesc_base{$key} = $d;
1088 };
1089 # throughput: (leaky bucket)
1090 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1091 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1092 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1093 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1094 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1095 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1096 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1097 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1098 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1099
1100 # pools: (pool of IO before throttling starts taking effect)
1101 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1102 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1103 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1104 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1105 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1106 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1107
1108 # burst lengths
1109 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1110 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1111 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1112 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1113 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1114 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1115
1116 # legacy support
1117 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1118 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1119 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1120 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1121
1122 my $ide_fmt = {
1123     %drivedesc_base,
1124     %model_fmt,
1125     %ssd_fmt,
1126     %wwn_fmt,
1127 };
1128 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1129
1130 my $idedesc = {
1131     optional => 1,
1132     type => 'string', format => $ide_fmt,
1133     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1134 };
1135 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1136
1137 my $scsi_fmt = {
1138     %drivedesc_base,
1139     %iothread_fmt,
1140     %queues_fmt,
1141     %scsiblock_fmt,
1142     %ssd_fmt,
1143     %wwn_fmt,
1144 };
1145 my $scsidesc = {
1146     optional => 1,
1147     type => 'string', format => $scsi_fmt,
1148     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1149 };
1150 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1151
1152 my $sata_fmt = {
1153     %drivedesc_base,
1154     %ssd_fmt,
1155     %wwn_fmt,
1156 };
1157 my $satadesc = {
1158     optional => 1,
1159     type => 'string', format => $sata_fmt,
1160     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1161 };
1162 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1163
1164 my $virtio_fmt = {
1165     %drivedesc_base,
1166     %iothread_fmt,
1167 };
1168 my $virtiodesc = {
1169     optional => 1,
1170     type => 'string', format => $virtio_fmt,
1171     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1172 };
1173 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1174
1175 my $alldrive_fmt = {
1176     %drivedesc_base,
1177     %iothread_fmt,
1178     %model_fmt,
1179     %queues_fmt,
1180     %scsiblock_fmt,
1181     %ssd_fmt,
1182     %wwn_fmt,
1183 };
1184
1185 my $efidisk_fmt = {
1186     volume => { alias => 'file' },
1187     file => {
1188         type => 'string',
1189         format => 'pve-volume-id-or-qm-path',
1190         default_key => 1,
1191         format_description => 'volume',
1192         description => "The drive's backing volume.",
1193     },
1194     format => get_standard_option('pve-qm-image-format'),
1195     size => {
1196         type => 'string',
1197         format => 'disk-size',
1198         format_description => 'DiskSize',
1199         description => "Disk size. This is purely informational and has no effect.",
1200         optional => 1,
1201     },
1202 };
1203
1204 my $efidisk_desc = {
1205     optional => 1,
1206     type => 'string', format => $efidisk_fmt,
1207     description => "Configure a Disk for storing EFI vars",
1208 };
1209
1210 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1211
1212 my $usb_fmt = {
1213     host => {
1214         default_key => 1,
1215         type => 'string', format => 'pve-qm-usb-device',
1216         format_description => 'HOSTUSBDEVICE|spice',
1217         description => <<EODESCR,
1218 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1219
1220  'bus-port(.port)*' (decimal numbers) or
1221  'vendor_id:product_id' (hexadeciaml numbers) or
1222  'spice'
1223
1224 You can use the 'lsusb -t' command to list existing usb devices.
1225
1226 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1227
1228 The value 'spice' can be used to add a usb redirection devices for spice.
1229 EODESCR
1230     },
1231     usb3 => {
1232         optional => 1,
1233         type => 'boolean',
1234         description => "Specifies whether if given host option is a USB3 device or port.",
1235         default => 0,
1236     },
1237 };
1238
1239 my $usbdesc = {
1240     optional => 1,
1241     type => 'string', format => $usb_fmt,
1242     description => "Configure an USB device (n is 0 to 4).",
1243 };
1244 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1245
1246 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1247 my $hostpci_fmt = {
1248     host => {
1249         default_key => 1,
1250         type => 'string',
1251         pattern => qr/$PCIRE(;$PCIRE)*/,
1252         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1253         description => <<EODESCR,
1254 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1255 of PCI virtual functions of the host. HOSTPCIID syntax is:
1256
1257 'bus:dev.func' (hexadecimal numbers)
1258
1259 You can us the 'lspci' command to list existing PCI devices.
1260 EODESCR
1261     },
1262     rombar => {
1263         type => 'boolean',
1264         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1265         optional => 1,
1266         default => 1,
1267     },
1268     romfile => {
1269         type => 'string',
1270         pattern => '[^,;]+',
1271         format_description => 'string',
1272         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1273         optional => 1,
1274     },
1275     pcie => {
1276         type => 'boolean',
1277         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1278         optional => 1,
1279         default => 0,
1280     },
1281     'x-vga' => {
1282         type => 'boolean',
1283         description =>  "Enable vfio-vga device support.",
1284         optional => 1,
1285         default => 0,
1286     },
1287     'mdev' => {
1288         type => 'string',
1289         format_description => 'string',
1290         pattern => '[^/\.:]+',
1291         optional => 1,
1292         description => <<EODESCR
1293 The type of mediated device to use.
1294 An instance of this type will be created on startup of the VM and
1295 will be cleaned up when the VM stops.
1296 EODESCR
1297     }
1298 };
1299 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1300
1301 my $hostpcidesc = {
1302         optional => 1,
1303         type => 'string', format => 'pve-qm-hostpci',
1304         description => "Map host PCI devices into guest.",
1305         verbose_description =>  <<EODESCR,
1306 Map host PCI devices into guest.
1307
1308 NOTE: This option allows direct access to host hardware. So it is no longer
1309 possible to migrate such machines - use with special care.
1310
1311 CAUTION: Experimental! User reported problems with this option.
1312 EODESCR
1313 };
1314 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1315
1316 my $serialdesc = {
1317         optional => 1,
1318         type => 'string',
1319         pattern => '(/dev/.+|socket)',
1320         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1321         verbose_description =>  <<EODESCR,
1322 Create a serial device inside the VM (n is 0 to 3), and pass through a
1323 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1324 host side (use 'qm terminal' to open a terminal connection).
1325
1326 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1327
1328 CAUTION: Experimental! User reported problems with this option.
1329 EODESCR
1330 };
1331
1332 my $paralleldesc= {
1333         optional => 1,
1334         type => 'string',
1335         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1336         description =>  "Map host parallel devices (n is 0 to 2).",
1337         verbose_description =>  <<EODESCR,
1338 Map host parallel devices (n is 0 to 2).
1339
1340 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1341
1342 CAUTION: Experimental! User reported problems with this option.
1343 EODESCR
1344 };
1345
1346 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1347     $confdesc->{"parallel$i"} = $paralleldesc;
1348 }
1349
1350 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1351     $confdesc->{"serial$i"} = $serialdesc;
1352 }
1353
1354 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1355     $confdesc->{"hostpci$i"} = $hostpcidesc;
1356 }
1357
1358 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1359     $drivename_hash->{"ide$i"} = 1;
1360     $confdesc->{"ide$i"} = $idedesc;
1361 }
1362
1363 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1364     $drivename_hash->{"sata$i"} = 1;
1365     $confdesc->{"sata$i"} = $satadesc;
1366 }
1367
1368 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1369     $drivename_hash->{"scsi$i"} = 1;
1370     $confdesc->{"scsi$i"} = $scsidesc ;
1371 }
1372
1373 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1374     $drivename_hash->{"virtio$i"} = 1;
1375     $confdesc->{"virtio$i"} = $virtiodesc;
1376 }
1377
1378 $drivename_hash->{efidisk0} = 1;
1379 $confdesc->{efidisk0} = $efidisk_desc;
1380
1381 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1382     $confdesc->{"usb$i"} = $usbdesc;
1383 }
1384
1385 my $unuseddesc = {
1386     optional => 1,
1387     type => 'string', format => 'pve-volume-id',
1388     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1389 };
1390
1391 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1392     $confdesc->{"unused$i"} = $unuseddesc;
1393 }
1394
1395 my $kvm_api_version = 0;
1396
1397 sub kvm_version {
1398     return $kvm_api_version if $kvm_api_version;
1399
1400     open my $fh, '<', '/dev/kvm'
1401         or return undef;
1402
1403     # 0xae00 => KVM_GET_API_VERSION
1404     $kvm_api_version = ioctl($fh, 0xae00, 0);
1405
1406     return $kvm_api_version;
1407 }
1408
1409 my $kvm_user_version = {};
1410 my $kvm_mtime = {};
1411
1412 sub kvm_user_version {
1413     my ($binary) = @_;
1414
1415     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1416     my $st = stat($binary);
1417
1418     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1419     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1420         $cachedmtime == $st->mtime;
1421
1422     $kvm_user_version->{$binary} = 'unknown';
1423     $kvm_mtime->{$binary} = $st->mtime;
1424
1425     my $code = sub {
1426         my $line = shift;
1427         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1428             $kvm_user_version->{$binary} = $2;
1429         }
1430     };
1431
1432     eval { run_command([$binary, '--version'], outfunc => $code); };
1433     warn $@ if $@;
1434
1435     return $kvm_user_version->{$binary};
1436
1437 }
1438
1439 sub kernel_has_vhost_net {
1440     return -c '/dev/vhost-net';
1441 }
1442
1443 sub valid_drive_names {
1444     # order is important - used to autoselect boot disk
1445     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1446             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1447             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1448             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1449             'efidisk0');
1450 }
1451
1452 sub is_valid_drivename {
1453     my $dev = shift;
1454
1455     return defined($drivename_hash->{$dev});
1456 }
1457
1458 sub option_exists {
1459     my $key = shift;
1460     return defined($confdesc->{$key});
1461 }
1462
1463 my $cdrom_path;
1464 sub get_cdrom_path {
1465
1466     return  $cdrom_path if $cdrom_path;
1467
1468     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1469     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1470     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1471 }
1472
1473 sub get_iso_path {
1474     my ($storecfg, $vmid, $cdrom) = @_;
1475
1476     if ($cdrom eq 'cdrom') {
1477         return get_cdrom_path();
1478     } elsif ($cdrom eq 'none') {
1479         return '';
1480     } elsif ($cdrom =~ m|^/|) {
1481         return $cdrom;
1482     } else {
1483         return PVE::Storage::path($storecfg, $cdrom);
1484     }
1485 }
1486
1487 # try to convert old style file names to volume IDs
1488 sub filename_to_volume_id {
1489     my ($vmid, $file, $media) = @_;
1490
1491      if (!($file eq 'none' || $file eq 'cdrom' ||
1492           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1493
1494         return undef if $file =~ m|/|;
1495
1496         if ($media && $media eq 'cdrom') {
1497             $file = "local:iso/$file";
1498         } else {
1499             $file = "local:$vmid/$file";
1500         }
1501     }
1502
1503     return $file;
1504 }
1505
1506 sub verify_media_type {
1507     my ($opt, $vtype, $media) = @_;
1508
1509     return if !$media;
1510
1511     my $etype;
1512     if ($media eq 'disk') {
1513         $etype = 'images';
1514     } elsif ($media eq 'cdrom') {
1515         $etype = 'iso';
1516     } else {
1517         die "internal error";
1518     }
1519
1520     return if ($vtype eq $etype);
1521
1522     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1523 }
1524
1525 sub cleanup_drive_path {
1526     my ($opt, $storecfg, $drive) = @_;
1527
1528     # try to convert filesystem paths to volume IDs
1529
1530     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1531         ($drive->{file} !~ m|^/dev/.+|) &&
1532         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1533         ($drive->{file} !~ m/^\d+$/)) {
1534         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1535         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1536         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1537         verify_media_type($opt, $vtype, $drive->{media});
1538         $drive->{file} = $volid;
1539     }
1540
1541     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1542 }
1543
1544 sub parse_hotplug_features {
1545     my ($data) = @_;
1546
1547     my $res = {};
1548
1549     return $res if $data eq '0';
1550
1551     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1552
1553     foreach my $feature (PVE::Tools::split_list($data)) {
1554         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1555             $res->{$1} = 1;
1556         } else {
1557             die "invalid hotplug feature '$feature'\n";
1558         }
1559     }
1560     return $res;
1561 }
1562
1563 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1564 sub pve_verify_hotplug_features {
1565     my ($value, $noerr) = @_;
1566
1567     return $value if parse_hotplug_features($value);
1568
1569     return undef if $noerr;
1570
1571     die "unable to parse hotplug option\n";
1572 }
1573
1574 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1575 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1576 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1577 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1578 #        [,iothread=on][,serial=serial][,model=model]
1579
1580 sub parse_drive {
1581     my ($key, $data) = @_;
1582
1583     my ($interface, $index);
1584
1585     if ($key =~ m/^([^\d]+)(\d+)$/) {
1586         $interface = $1;
1587         $index = $2;
1588     } else {
1589         return undef;
1590     }
1591
1592     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1593                                      : $confdesc->{$key}->{format};
1594     if (!$desc) {
1595         warn "invalid drive key: $key\n";
1596         return undef;
1597     }
1598     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1599     return undef if !$res;
1600     $res->{interface} = $interface;
1601     $res->{index} = $index;
1602
1603     my $error = 0;
1604     foreach my $opt (qw(bps bps_rd bps_wr)) {
1605         if (my $bps = defined(delete $res->{$opt})) {
1606             if (defined($res->{"m$opt"})) {
1607                 warn "both $opt and m$opt specified\n";
1608                 ++$error;
1609                 next;
1610             }
1611             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1612         }
1613     }
1614
1615     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1616     for my $requirement (
1617         [mbps_max => 'mbps'],
1618         [mbps_rd_max => 'mbps_rd'],
1619         [mbps_wr_max => 'mbps_wr'],
1620         [miops_max => 'miops'],
1621         [miops_rd_max => 'miops_rd'],
1622         [miops_wr_max => 'miops_wr'],
1623         [bps_max_length => 'mbps_max'],
1624         [bps_rd_max_length => 'mbps_rd_max'],
1625         [bps_wr_max_length => 'mbps_wr_max'],
1626         [iops_max_length => 'iops_max'],
1627         [iops_rd_max_length => 'iops_rd_max'],
1628         [iops_wr_max_length => 'iops_wr_max']) {
1629         my ($option, $requires) = @$requirement;
1630         if ($res->{$option} && !$res->{$requires}) {
1631             warn "$option requires $requires\n";
1632             ++$error;
1633         }
1634     }
1635
1636     return undef if $error;
1637
1638     return undef if $res->{mbps_rd} && $res->{mbps};
1639     return undef if $res->{mbps_wr} && $res->{mbps};
1640     return undef if $res->{iops_rd} && $res->{iops};
1641     return undef if $res->{iops_wr} && $res->{iops};
1642
1643     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1644         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1645         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1646         return undef if $res->{interface} eq 'virtio';
1647     }
1648
1649     if (my $size = $res->{size}) {
1650         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1651     }
1652
1653     return $res;
1654 }
1655
1656 sub print_drive {
1657     my ($drive) = @_;
1658     my $data = { %$drive };
1659     delete $data->{$_} for qw(index interface);
1660     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1661 }
1662
1663 sub scsi_inquiry {
1664     my($fh, $noerr) = @_;
1665
1666     my $SG_IO = 0x2285;
1667     my $SG_GET_VERSION_NUM = 0x2282;
1668
1669     my $versionbuf = "\x00" x 8;
1670     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1671     if (!$ret) {
1672         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1673         return undef;
1674     }
1675     my $version = unpack("I", $versionbuf);
1676     if ($version < 30000) {
1677         die "scsi generic interface too old\n"  if !$noerr;
1678         return undef;
1679     }
1680
1681     my $buf = "\x00" x 36;
1682     my $sensebuf = "\x00" x 8;
1683     my $cmd = pack("C x3 C x1", 0x12, 36);
1684
1685     # see /usr/include/scsi/sg.h
1686     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1687
1688     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1689                       length($sensebuf), 0, length($buf), $buf,
1690                       $cmd, $sensebuf, 6000);
1691
1692     $ret = ioctl($fh, $SG_IO, $packet);
1693     if (!$ret) {
1694         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1695         return undef;
1696     }
1697
1698     my @res = unpack($sg_io_hdr_t, $packet);
1699     if ($res[17] || $res[18]) {
1700         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1701         return undef;
1702     }
1703
1704     my $res = {};
1705     (my $byte0, my $byte1, $res->{vendor},
1706      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1707
1708     $res->{removable} = $byte1 & 128 ? 1 : 0;
1709     $res->{type} = $byte0 & 31;
1710
1711     return $res;
1712 }
1713
1714 sub path_is_scsi {
1715     my ($path) = @_;
1716
1717     my $fh = IO::File->new("+<$path") || return undef;
1718     my $res = scsi_inquiry($fh, 1);
1719     close($fh);
1720
1721     return $res;
1722 }
1723
1724 sub print_tabletdevice_full {
1725     my ($conf, $arch) = @_;
1726
1727     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1728
1729     # we use uhci for old VMs because tablet driver was buggy in older qemu
1730     my $usbbus;
1731     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1732         $usbbus = 'ehci';
1733     } else {
1734         $usbbus = 'uhci';
1735     }
1736
1737     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1738 }
1739
1740 sub print_keyboarddevice_full {
1741     my ($conf, $arch, $machine) = @_;
1742
1743     return undef if $arch ne 'aarch64';
1744
1745     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1746 }
1747
1748 sub print_drivedevice_full {
1749     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1750
1751     my $device = '';
1752     my $maxdev = 0;
1753
1754     if ($drive->{interface} eq 'virtio') {
1755         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1756         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1757         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1758     } elsif ($drive->{interface} eq 'scsi') {
1759
1760         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1761         my $unit = $drive->{index} % $maxdev;
1762         my $devicetype = 'hd';
1763         my $path = '';
1764         if (drive_is_cdrom($drive)) {
1765             $devicetype = 'cd';
1766         } else {
1767             if ($drive->{file} =~ m|^/|) {
1768                 $path = $drive->{file};
1769                 if (my $info = path_is_scsi($path)) {
1770                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1771                         $devicetype = 'block';
1772                     } elsif ($info->{type} == 1) { # tape
1773                         $devicetype = 'generic';
1774                     }
1775                 }
1776             } else {
1777                  $path = PVE::Storage::path($storecfg, $drive->{file});
1778             }
1779
1780             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1781             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1782             if ($path =~ m/^iscsi\:\/\// &&
1783                !min_version($version, 4, 1)) {
1784                 $devicetype = 'generic';
1785             }
1786         }
1787
1788         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1789             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1790         } else {
1791             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1792         }
1793
1794         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1795             $device .= ",rotation_rate=1";
1796         }
1797         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1798
1799     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1800         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1801         my $controller = int($drive->{index} / $maxdev);
1802         my $unit = $drive->{index} % $maxdev;
1803         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1804
1805         $device = "ide-$devicetype";
1806         if ($drive->{interface} eq 'ide') {
1807             $device .= ",bus=ide.$controller,unit=$unit";
1808         } else {
1809             $device .= ",bus=ahci$controller.$unit";
1810         }
1811         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1812
1813         if ($devicetype eq 'hd') {
1814             if (my $model = $drive->{model}) {
1815                 $model = URI::Escape::uri_unescape($model);
1816                 $device .= ",model=$model";
1817             }
1818             if ($drive->{ssd}) {
1819                 $device .= ",rotation_rate=1";
1820             }
1821         }
1822         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1823     } elsif ($drive->{interface} eq 'usb') {
1824         die "implement me";
1825         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1826     } else {
1827         die "unsupported interface type";
1828     }
1829
1830     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1831
1832     if (my $serial = $drive->{serial}) {
1833         $serial = URI::Escape::uri_unescape($serial);
1834         $device .= ",serial=$serial";
1835     }
1836
1837
1838     return $device;
1839 }
1840
1841 sub get_initiator_name {
1842     my $initiator;
1843
1844     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1845     while (defined(my $line = <$fh>)) {
1846         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1847         $initiator = $1;
1848         last;
1849     }
1850     $fh->close();
1851
1852     return $initiator;
1853 }
1854
1855 sub print_drive_full {
1856     my ($storecfg, $vmid, $drive) = @_;
1857
1858     my $path;
1859     my $volid = $drive->{file};
1860     my $format;
1861
1862     if (drive_is_cdrom($drive)) {
1863         $path = get_iso_path($storecfg, $vmid, $volid);
1864     } else {
1865         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1866         if ($storeid) {
1867             $path = PVE::Storage::path($storecfg, $volid);
1868             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1869             $format = qemu_img_format($scfg, $volname);
1870         } else {
1871             $path = $volid;
1872             $format = "raw";
1873         }
1874    }
1875
1876     my $opts = '';
1877     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1878     foreach my $o (@qemu_drive_options) {
1879         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1880     }
1881
1882     # snapshot only accepts on|off
1883     if (defined($drive->{snapshot})) {
1884         my $v = $drive->{snapshot} ? 'on' : 'off';
1885         $opts .= ",snapshot=$v";
1886     }
1887
1888     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1889         my ($dir, $qmpname) = @$type;
1890         if (my $v = $drive->{"mbps$dir"}) {
1891             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1892         }
1893         if (my $v = $drive->{"mbps${dir}_max"}) {
1894             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1895         }
1896         if (my $v = $drive->{"bps${dir}_max_length"}) {
1897             $opts .= ",throttling.bps$qmpname-max-length=$v";
1898         }
1899         if (my $v = $drive->{"iops${dir}"}) {
1900             $opts .= ",throttling.iops$qmpname=$v";
1901         }
1902         if (my $v = $drive->{"iops${dir}_max"}) {
1903             $opts .= ",throttling.iops$qmpname-max=$v";
1904         }
1905         if (my $v = $drive->{"iops${dir}_max_length"}) {
1906             $opts .= ",throttling.iops$qmpname-max-length=$v";
1907         }
1908     }
1909
1910     $opts .= ",format=$format" if $format && !$drive->{format};
1911
1912     my $cache_direct = 0;
1913
1914     if (my $cache = $drive->{cache}) {
1915         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1916     } elsif (!drive_is_cdrom($drive)) {
1917         $opts .= ",cache=none";
1918         $cache_direct = 1;
1919     }
1920
1921     # aio native works only with O_DIRECT
1922     if (!$drive->{aio}) {
1923         if($cache_direct) {
1924             $opts .= ",aio=native";
1925         } else {
1926             $opts .= ",aio=threads";
1927         }
1928     }
1929
1930     if (!drive_is_cdrom($drive)) {
1931         my $detectzeroes;
1932         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1933             $detectzeroes = 'off';
1934         } elsif ($drive->{discard}) {
1935             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1936         } else {
1937             # This used to be our default with discard not being specified:
1938             $detectzeroes = 'on';
1939         }
1940         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1941     }
1942
1943     my $pathinfo = $path ? "file=$path," : '';
1944
1945     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1946 }
1947
1948 sub print_netdevice_full {
1949     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1950
1951     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1952
1953     my $device = $net->{model};
1954     if ($net->{model} eq 'virtio') {
1955          $device = 'virtio-net-pci';
1956      };
1957
1958     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1959     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1960     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1961         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1962         my $vectors = $net->{queues} * 2 + 2;
1963         $tmpstr .= ",vectors=$vectors,mq=on";
1964     }
1965     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1966
1967     if ($use_old_bios_files) {
1968         my $romfile;
1969         if ($device eq 'virtio-net-pci') {
1970             $romfile = 'pxe-virtio.rom';
1971         } elsif ($device eq 'e1000') {
1972             $romfile = 'pxe-e1000.rom';
1973         } elsif ($device eq 'ne2k') {
1974             $romfile = 'pxe-ne2k_pci.rom';
1975         } elsif ($device eq 'pcnet') {
1976             $romfile = 'pxe-pcnet.rom';
1977         } elsif ($device eq 'rtl8139') {
1978             $romfile = 'pxe-rtl8139.rom';
1979         }
1980         $tmpstr .= ",romfile=$romfile" if $romfile;
1981     }
1982
1983     return $tmpstr;
1984 }
1985
1986 sub print_netdev_full {
1987     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1988
1989     my $i = '';
1990     if ($netid =~ m/^net(\d+)$/) {
1991         $i = int($1);
1992     }
1993
1994     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1995
1996     my $ifname = "tap${vmid}i$i";
1997
1998     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1999     die "interface name '$ifname' is too long (max 15 character)\n"
2000         if length($ifname) >= 16;
2001
2002     my $vhostparam = '';
2003     if (is_native($arch)) {
2004         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2005     }
2006
2007     my $vmname = $conf->{name} || "vm$vmid";
2008
2009     my $netdev = "";
2010     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2011
2012     if ($net->{bridge}) {
2013         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2014     } else {
2015         $netdev = "type=user,id=$netid,hostname=$vmname";
2016     }
2017
2018     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2019
2020     return $netdev;
2021 }
2022
2023 my $vga_map = {
2024     'cirrus' => 'cirrus-vga',
2025     'std' => 'VGA',
2026     'vmware' => 'vmware-svga',
2027     'virtio' => 'virtio-vga',
2028 };
2029
2030 sub print_vga_device {
2031     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2032
2033     my $type = $vga_map->{$vga->{type}};
2034     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2035         $type = 'virtio-gpu';
2036     }
2037     my $vgamem_mb = $vga->{memory};
2038
2039     my $max_outputs = '';
2040     if ($qxlnum) {
2041         $type = $id ? 'qxl' : 'qxl-vga';
2042
2043         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2044             # set max outputs so linux can have up to 4 qxl displays with one device
2045             if (min_version($machine_version, 4, 1)) {
2046                 $max_outputs = ",max_outputs=4";
2047             }
2048         }
2049     }
2050
2051     die "no devicetype for $vga->{type}\n" if !$type;
2052
2053     my $memory = "";
2054     if ($vgamem_mb) {
2055         if ($vga->{type} eq 'virtio') {
2056             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2057             $memory = ",max_hostmem=$bytes";
2058         } elsif ($qxlnum) {
2059             # from https://www.spice-space.org/multiple-monitors.html
2060             $memory = ",vgamem_mb=$vga->{memory}";
2061             my $ram = $vgamem_mb * 4;
2062             my $vram = $vgamem_mb * 2;
2063             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2064         } else {
2065             $memory = ",vgamem_mb=$vga->{memory}";
2066         }
2067     } elsif ($qxlnum && $id) {
2068         $memory = ",ram_size=67108864,vram_size=33554432";
2069     }
2070
2071     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2072     my $vgaid = "vga" . ($id // '');
2073     my $pciaddr;
2074
2075     if ($q35 && $vgaid eq 'vga') {
2076         # the first display uses pcie.0 bus on q35 machines
2077         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2078     } else {
2079         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2080     }
2081
2082     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2083 }
2084
2085 sub drive_is_cloudinit {
2086     my ($drive) = @_;
2087     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2088 }
2089
2090 sub drive_is_cdrom {
2091     my ($drive, $exclude_cloudinit) = @_;
2092
2093     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2094
2095     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2096
2097 }
2098
2099 sub parse_number_sets {
2100     my ($set) = @_;
2101     my $res = [];
2102     foreach my $part (split(/;/, $set)) {
2103         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2104             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2105             push @$res, [ $1, $2 ];
2106         } else {
2107             die "invalid range: $part\n";
2108         }
2109     }
2110     return $res;
2111 }
2112
2113 sub parse_numa {
2114     my ($data) = @_;
2115
2116     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2117     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2118     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2119     return $res;
2120 }
2121
2122 sub parse_hostpci {
2123     my ($value) = @_;
2124
2125     return undef if !$value;
2126
2127     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2128
2129     my @idlist = split(/;/, $res->{host});
2130     delete $res->{host};
2131     foreach my $id (@idlist) {
2132         my $devs = PVE::SysFSTools::lspci($id);
2133         die "no PCI device found for '$id'\n" if !scalar(@$devs);
2134         push @{$res->{pciid}}, @$devs;
2135     }
2136     return $res;
2137 }
2138
2139 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2140 sub parse_net {
2141     my ($data) = @_;
2142
2143     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2144     if ($@) {
2145         warn $@;
2146         return undef;
2147     }
2148     if (!defined($res->{macaddr})) {
2149         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2150         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2151     }
2152     return $res;
2153 }
2154
2155 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2156 sub parse_ipconfig {
2157     my ($data) = @_;
2158
2159     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2160     if ($@) {
2161         warn $@;
2162         return undef;
2163     }
2164
2165     if ($res->{gw} && !$res->{ip}) {
2166         warn 'gateway specified without specifying an IP address';
2167         return undef;
2168     }
2169     if ($res->{gw6} && !$res->{ip6}) {
2170         warn 'IPv6 gateway specified without specifying an IPv6 address';
2171         return undef;
2172     }
2173     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2174         warn 'gateway specified together with DHCP';
2175         return undef;
2176     }
2177     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2178         # gw6 + auto/dhcp
2179         warn "IPv6 gateway specified together with $res->{ip6} address";
2180         return undef;
2181     }
2182
2183     if (!$res->{ip} && !$res->{ip6}) {
2184         return { ip => 'dhcp', ip6 => 'dhcp' };
2185     }
2186
2187     return $res;
2188 }
2189
2190 sub print_net {
2191     my $net = shift;
2192
2193     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2194 }
2195
2196 sub add_random_macs {
2197     my ($settings) = @_;
2198
2199     foreach my $opt (keys %$settings) {
2200         next if $opt !~ m/^net(\d+)$/;
2201         my $net = parse_net($settings->{$opt});
2202         next if !$net;
2203         $settings->{$opt} = print_net($net);
2204     }
2205 }
2206
2207 sub vm_is_volid_owner {
2208     my ($storecfg, $vmid, $volid) = @_;
2209
2210     if ($volid !~  m|^/|) {
2211         my ($path, $owner);
2212         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2213         if ($owner && ($owner == $vmid)) {
2214             return 1;
2215         }
2216     }
2217
2218     return undef;
2219 }
2220
2221 sub vmconfig_register_unused_drive {
2222     my ($storecfg, $vmid, $conf, $drive) = @_;
2223
2224     if (drive_is_cloudinit($drive)) {
2225         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2226         warn $@ if $@;
2227     } elsif (!drive_is_cdrom($drive)) {
2228         my $volid = $drive->{file};
2229         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2230             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2231         }
2232     }
2233 }
2234
2235 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2236 my $smbios1_fmt = {
2237     uuid => {
2238         type => 'string',
2239         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2240         format_description => 'UUID',
2241         description => "Set SMBIOS1 UUID.",
2242         optional => 1,
2243     },
2244     version => {
2245         type => 'string',
2246         pattern => '[A-Za-z0-9+\/]+={0,2}',
2247         format_description => 'Base64 encoded string',
2248         description => "Set SMBIOS1 version.",
2249         optional => 1,
2250     },
2251     serial => {
2252         type => 'string',
2253         pattern => '[A-Za-z0-9+\/]+={0,2}',
2254         format_description => 'Base64 encoded string',
2255         description => "Set SMBIOS1 serial number.",
2256         optional => 1,
2257     },
2258     manufacturer => {
2259         type => 'string',
2260         pattern => '[A-Za-z0-9+\/]+={0,2}',
2261         format_description => 'Base64 encoded string',
2262         description => "Set SMBIOS1 manufacturer.",
2263         optional => 1,
2264     },
2265     product => {
2266         type => 'string',
2267         pattern => '[A-Za-z0-9+\/]+={0,2}',
2268         format_description => 'Base64 encoded string',
2269         description => "Set SMBIOS1 product ID.",
2270         optional => 1,
2271     },
2272     sku => {
2273         type => 'string',
2274         pattern => '[A-Za-z0-9+\/]+={0,2}',
2275         format_description => 'Base64 encoded string',
2276         description => "Set SMBIOS1 SKU string.",
2277         optional => 1,
2278     },
2279     family => {
2280         type => 'string',
2281         pattern => '[A-Za-z0-9+\/]+={0,2}',
2282         format_description => 'Base64 encoded string',
2283         description => "Set SMBIOS1 family string.",
2284         optional => 1,
2285     },
2286     base64 => {
2287         type => 'boolean',
2288         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2289         optional => 1,
2290     },
2291 };
2292
2293 sub parse_smbios1 {
2294     my ($data) = @_;
2295
2296     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2297     warn $@ if $@;
2298     return $res;
2299 }
2300
2301 sub print_smbios1 {
2302     my ($smbios1) = @_;
2303     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2304 }
2305
2306 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2307
2308 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2309 sub verify_bootdisk {
2310     my ($value, $noerr) = @_;
2311
2312     return $value if is_valid_drivename($value);
2313
2314     return undef if $noerr;
2315
2316     die "invalid boot disk '$value'\n";
2317 }
2318
2319 sub parse_watchdog {
2320     my ($value) = @_;
2321
2322     return undef if !$value;
2323
2324     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2325     warn $@ if $@;
2326     return $res;
2327 }
2328
2329 sub parse_guest_agent {
2330     my ($value) = @_;
2331
2332     return {} if !defined($value->{agent});
2333
2334     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2335     warn $@ if $@;
2336
2337     # if the agent is disabled ignore the other potentially set properties
2338     return {} if !$res->{enabled};
2339     return $res;
2340 }
2341
2342 sub parse_vga {
2343     my ($value) = @_;
2344
2345     return {} if !$value;
2346     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2347     warn $@ if $@;
2348     return $res;
2349 }
2350
2351 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2352 sub verify_usb_device {
2353     my ($value, $noerr) = @_;
2354
2355     return $value if parse_usb_device($value);
2356
2357     return undef if $noerr;
2358
2359     die "unable to parse usb device\n";
2360 }
2361
2362 # add JSON properties for create and set function
2363 sub json_config_properties {
2364     my $prop = shift;
2365
2366     foreach my $opt (keys %$confdesc) {
2367         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2368         $prop->{$opt} = $confdesc->{$opt};
2369     }
2370
2371     return $prop;
2372 }
2373
2374 # return copy of $confdesc_cloudinit to generate documentation
2375 sub cloudinit_config_properties {
2376
2377     return dclone($confdesc_cloudinit);
2378 }
2379
2380 sub check_type {
2381     my ($key, $value) = @_;
2382
2383     die "unknown setting '$key'\n" if !$confdesc->{$key};
2384
2385     my $type = $confdesc->{$key}->{type};
2386
2387     if (!defined($value)) {
2388         die "got undefined value\n";
2389     }
2390
2391     if ($value =~ m/[\n\r]/) {
2392         die "property contains a line feed\n";
2393     }
2394
2395     if ($type eq 'boolean') {
2396         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2397         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2398         die "type check ('boolean') failed - got '$value'\n";
2399     } elsif ($type eq 'integer') {
2400         return int($1) if $value =~ m/^(\d+)$/;
2401         die "type check ('integer') failed - got '$value'\n";
2402     } elsif ($type eq 'number') {
2403         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2404         die "type check ('number') failed - got '$value'\n";
2405     } elsif ($type eq 'string') {
2406         if (my $fmt = $confdesc->{$key}->{format}) {
2407             PVE::JSONSchema::check_format($fmt, $value);
2408             return $value;
2409         }
2410         $value =~ s/^\"(.*)\"$/$1/;
2411         return $value;
2412     } else {
2413         die "internal error"
2414     }
2415 }
2416
2417 sub destroy_vm {
2418     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2419
2420     my $conf = PVE::QemuConfig->load_config($vmid);
2421
2422     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2423
2424     if ($conf->{template}) {
2425         # check if any base image is still used by a linked clone
2426         foreach_drive($conf, sub {
2427                 my ($ds, $drive) = @_;
2428                 return if drive_is_cdrom($drive);
2429
2430                 my $volid = $drive->{file};
2431                 return if !$volid || $volid =~ m|^/|;
2432
2433                 die "base volume '$volid' is still in use by linked cloned\n"
2434                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2435
2436         });
2437     }
2438
2439     # only remove disks owned by this VM
2440     foreach_drive($conf, sub {
2441         my ($ds, $drive) = @_;
2442         return if drive_is_cdrom($drive, 1);
2443
2444         my $volid = $drive->{file};
2445         return if !$volid || $volid =~ m|^/|;
2446
2447         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2448         return if !$path || !$owner || ($owner != $vmid);
2449
2450         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2451         warn "Could not remove disk '$volid', check manually: $@" if $@;
2452     });
2453
2454     # also remove unused disk
2455     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2456     PVE::Storage::foreach_volid($vmdisks, sub {
2457         my ($volid, $sid, $volname, $d) = @_;
2458         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2459         warn $@ if $@;
2460     });
2461
2462     if (defined $replacement_conf) {
2463         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2464     } else {
2465         PVE::QemuConfig->destroy_config($vmid);
2466     }
2467 }
2468
2469 sub parse_vm_config {
2470     my ($filename, $raw) = @_;
2471
2472     return undef if !defined($raw);
2473
2474     my $res = {
2475         digest => Digest::SHA::sha1_hex($raw),
2476         snapshots => {},
2477         pending => {},
2478     };
2479
2480     $filename =~ m|/qemu-server/(\d+)\.conf$|
2481         || die "got strange filename '$filename'";
2482
2483     my $vmid = $1;
2484
2485     my $conf = $res;
2486     my $descr;
2487     my $section = '';
2488
2489     my @lines = split(/\n/, $raw);
2490     foreach my $line (@lines) {
2491         next if $line =~ m/^\s*$/;
2492
2493         if ($line =~ m/^\[PENDING\]\s*$/i) {
2494             $section = 'pending';
2495             if (defined($descr)) {
2496                 $descr =~ s/\s+$//;
2497                 $conf->{description} = $descr;
2498             }
2499             $descr = undef;
2500             $conf = $res->{$section} = {};
2501             next;
2502
2503         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2504             $section = $1;
2505             if (defined($descr)) {
2506                 $descr =~ s/\s+$//;
2507                 $conf->{description} = $descr;
2508             }
2509             $descr = undef;
2510             $conf = $res->{snapshots}->{$section} = {};
2511             next;
2512         }
2513
2514         if ($line =~ m/^\#(.*)\s*$/) {
2515             $descr = '' if !defined($descr);
2516             $descr .= PVE::Tools::decode_text($1) . "\n";
2517             next;
2518         }
2519
2520         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2521             $descr = '' if !defined($descr);
2522             $descr .= PVE::Tools::decode_text($2);
2523         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2524             $conf->{snapstate} = $1;
2525         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2526             my $key = $1;
2527             my $value = $2;
2528             $conf->{$key} = $value;
2529         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2530             my $value = $1;
2531             if ($section eq 'pending') {
2532                 $conf->{delete} = $value; # we parse this later
2533             } else {
2534                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2535             }
2536         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2537             my $key = $1;
2538             my $value = $2;
2539             eval { $value = check_type($key, $value); };
2540             if ($@) {
2541                 warn "vm $vmid - unable to parse value of '$key' - $@";
2542             } else {
2543                 $key = 'ide2' if $key eq 'cdrom';
2544                 my $fmt = $confdesc->{$key}->{format};
2545                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2546                     my $v = parse_drive($key, $value);
2547                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2548                         $v->{file} = $volid;
2549                         $value = print_drive($v);
2550                     } else {
2551                         warn "vm $vmid - unable to parse value of '$key'\n";
2552                         next;
2553                     }
2554                 }
2555
2556                 $conf->{$key} = $value;
2557             }
2558         }
2559     }
2560
2561     if (defined($descr)) {
2562         $descr =~ s/\s+$//;
2563         $conf->{description} = $descr;
2564     }
2565     delete $res->{snapstate}; # just to be sure
2566
2567     return $res;
2568 }
2569
2570 sub write_vm_config {
2571     my ($filename, $conf) = @_;
2572
2573     delete $conf->{snapstate}; # just to be sure
2574
2575     if ($conf->{cdrom}) {
2576         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2577         $conf->{ide2} = $conf->{cdrom};
2578         delete $conf->{cdrom};
2579     }
2580
2581     # we do not use 'smp' any longer
2582     if ($conf->{sockets}) {
2583         delete $conf->{smp};
2584     } elsif ($conf->{smp}) {
2585         $conf->{sockets} = $conf->{smp};
2586         delete $conf->{cores};
2587         delete $conf->{smp};
2588     }
2589
2590     my $used_volids = {};
2591
2592     my $cleanup_config = sub {
2593         my ($cref, $pending, $snapname) = @_;
2594
2595         foreach my $key (keys %$cref) {
2596             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2597                 $key eq 'snapstate' || $key eq 'pending';
2598             my $value = $cref->{$key};
2599             if ($key eq 'delete') {
2600                 die "propertry 'delete' is only allowed in [PENDING]\n"
2601                     if !$pending;
2602                 # fixme: check syntax?
2603                 next;
2604             }
2605             eval { $value = check_type($key, $value); };
2606             die "unable to parse value of '$key' - $@" if $@;
2607
2608             $cref->{$key} = $value;
2609
2610             if (!$snapname && is_valid_drivename($key)) {
2611                 my $drive = parse_drive($key, $value);
2612                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2613             }
2614         }
2615     };
2616
2617     &$cleanup_config($conf);
2618
2619     &$cleanup_config($conf->{pending}, 1);
2620
2621     foreach my $snapname (keys %{$conf->{snapshots}}) {
2622         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2623         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2624     }
2625
2626     # remove 'unusedX' settings if we re-add a volume
2627     foreach my $key (keys %$conf) {
2628         my $value = $conf->{$key};
2629         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2630             delete $conf->{$key};
2631         }
2632     }
2633
2634     my $generate_raw_config = sub {
2635         my ($conf, $pending) = @_;
2636
2637         my $raw = '';
2638
2639         # add description as comment to top of file
2640         if (defined(my $descr = $conf->{description})) {
2641             if ($descr) {
2642                 foreach my $cl (split(/\n/, $descr)) {
2643                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2644                 }
2645             } else {
2646                 $raw .= "#\n" if $pending;
2647             }
2648         }
2649
2650         foreach my $key (sort keys %$conf) {
2651             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2652             $raw .= "$key: $conf->{$key}\n";
2653         }
2654         return $raw;
2655     };
2656
2657     my $raw = &$generate_raw_config($conf);
2658
2659     if (scalar(keys %{$conf->{pending}})){
2660         $raw .= "\n[PENDING]\n";
2661         $raw .= &$generate_raw_config($conf->{pending}, 1);
2662     }
2663
2664     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2665         $raw .= "\n[$snapname]\n";
2666         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2667     }
2668
2669     return $raw;
2670 }
2671
2672 sub load_defaults {
2673
2674     my $res = {};
2675
2676     # we use static defaults from our JSON schema configuration
2677     foreach my $key (keys %$confdesc) {
2678         if (defined(my $default = $confdesc->{$key}->{default})) {
2679             $res->{$key} = $default;
2680         }
2681     }
2682
2683     return $res;
2684 }
2685
2686 sub config_list {
2687     my $vmlist = PVE::Cluster::get_vmlist();
2688     my $res = {};
2689     return $res if !$vmlist || !$vmlist->{ids};
2690     my $ids = $vmlist->{ids};
2691     my $nodename = nodename();
2692
2693     foreach my $vmid (keys %$ids) {
2694         my $d = $ids->{$vmid};
2695         next if !$d->{node} || $d->{node} ne $nodename;
2696         next if !$d->{type} || $d->{type} ne 'qemu';
2697         $res->{$vmid}->{exists} = 1;
2698     }
2699     return $res;
2700 }
2701
2702 # test if VM uses local resources (to prevent migration)
2703 sub check_local_resources {
2704     my ($conf, $noerr) = @_;
2705
2706     my @loc_res = ();
2707
2708     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2709     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2710
2711     push @loc_res, "ivshmem" if $conf->{ivshmem};
2712
2713     foreach my $k (keys %$conf) {
2714         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2715         # sockets are safe: they will recreated be on the target side post-migrate
2716         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2717         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2718     }
2719
2720     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2721
2722     return \@loc_res;
2723 }
2724
2725 # check if used storages are available on all nodes (use by migrate)
2726 sub check_storage_availability {
2727     my ($storecfg, $conf, $node) = @_;
2728
2729     foreach_drive($conf, sub {
2730         my ($ds, $drive) = @_;
2731
2732         my $volid = $drive->{file};
2733         return if !$volid;
2734
2735         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2736         return if !$sid;
2737
2738         # check if storage is available on both nodes
2739         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2740         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2741    });
2742 }
2743
2744 # list nodes where all VM images are available (used by has_feature API)
2745 sub shared_nodes {
2746     my ($conf, $storecfg) = @_;
2747
2748     my $nodelist = PVE::Cluster::get_nodelist();
2749     my $nodehash = { map { $_ => 1 } @$nodelist };
2750     my $nodename = nodename();
2751
2752     foreach_drive($conf, sub {
2753         my ($ds, $drive) = @_;
2754
2755         my $volid = $drive->{file};
2756         return if !$volid;
2757
2758         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2759         if ($storeid) {
2760             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2761             if ($scfg->{disable}) {
2762                 $nodehash = {};
2763             } elsif (my $avail = $scfg->{nodes}) {
2764                 foreach my $node (keys %$nodehash) {
2765                     delete $nodehash->{$node} if !$avail->{$node};
2766                 }
2767             } elsif (!$scfg->{shared}) {
2768                 foreach my $node (keys %$nodehash) {
2769                     delete $nodehash->{$node} if $node ne $nodename
2770                 }
2771             }
2772         }
2773     });
2774
2775     return $nodehash
2776 }
2777
2778 sub check_local_storage_availability {
2779     my ($conf, $storecfg) = @_;
2780
2781     my $nodelist = PVE::Cluster::get_nodelist();
2782     my $nodehash = { map { $_ => {} } @$nodelist };
2783
2784     foreach_drive($conf, sub {
2785         my ($ds, $drive) = @_;
2786
2787         my $volid = $drive->{file};
2788         return if !$volid;
2789
2790         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2791         if ($storeid) {
2792             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2793
2794             if ($scfg->{disable}) {
2795                 foreach my $node (keys %$nodehash) {
2796                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2797                 }
2798             } elsif (my $avail = $scfg->{nodes}) {
2799                 foreach my $node (keys %$nodehash) {
2800                     if (!$avail->{$node}) {
2801                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2802                     }
2803                 }
2804             }
2805         }
2806     });
2807
2808     foreach my $node (values %$nodehash) {
2809         if (my $unavail = $node->{unavailable_storages}) {
2810             $node->{unavailable_storages} = [ sort keys %$unavail ];
2811         }
2812     }
2813
2814     return $nodehash
2815 }
2816
2817 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2818 sub check_running {
2819     my ($vmid, $nocheck, $node) = @_;
2820
2821     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2822     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2823 }
2824
2825 sub vzlist {
2826
2827     my $vzlist = config_list();
2828
2829     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2830
2831     while (defined(my $de = $fd->read)) {
2832         next if $de !~ m/^(\d+)\.pid$/;
2833         my $vmid = $1;
2834         next if !defined($vzlist->{$vmid});
2835         if (my $pid = check_running($vmid)) {
2836             $vzlist->{$vmid}->{pid} = $pid;
2837         }
2838     }
2839
2840     return $vzlist;
2841 }
2842
2843 sub disksize {
2844     my ($storecfg, $conf) = @_;
2845
2846     my $bootdisk = $conf->{bootdisk};
2847     return undef if !$bootdisk;
2848     return undef if !is_valid_drivename($bootdisk);
2849
2850     return undef if !$conf->{$bootdisk};
2851
2852     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2853     return undef if !defined($drive);
2854
2855     return undef if drive_is_cdrom($drive);
2856
2857     my $volid = $drive->{file};
2858     return undef if !$volid;
2859
2860     return $drive->{size};
2861 }
2862
2863 our $vmstatus_return_properties = {
2864     vmid => get_standard_option('pve-vmid'),
2865     status => {
2866         description => "Qemu process status.",
2867         type => 'string',
2868         enum => ['stopped', 'running'],
2869     },
2870     maxmem => {
2871         description => "Maximum memory in bytes.",
2872         type => 'integer',
2873         optional => 1,
2874         renderer => 'bytes',
2875     },
2876     maxdisk => {
2877         description => "Root disk size in bytes.",
2878         type => 'integer',
2879         optional => 1,
2880         renderer => 'bytes',
2881     },
2882     name => {
2883         description => "VM name.",
2884         type => 'string',
2885         optional => 1,
2886     },
2887     qmpstatus => {
2888         description => "Qemu QMP agent status.",
2889         type => 'string',
2890         optional => 1,
2891     },
2892     pid => {
2893         description => "PID of running qemu process.",
2894         type => 'integer',
2895         optional => 1,
2896     },
2897     uptime => {
2898         description => "Uptime.",
2899         type => 'integer',
2900         optional => 1,
2901         renderer => 'duration',
2902     },
2903     cpus => {
2904         description => "Maximum usable CPUs.",
2905         type => 'number',
2906         optional => 1,
2907     },
2908     lock => {
2909         description => "The current config lock, if any.",
2910         type => 'string',
2911         optional => 1,
2912     },
2913     tags => {
2914         description  => "The current configured tags, if any",
2915         type => 'string',
2916         optional => 1,
2917     },
2918 };
2919
2920 my $last_proc_pid_stat;
2921
2922 # get VM status information
2923 # This must be fast and should not block ($full == false)
2924 # We only query KVM using QMP if $full == true (this can be slow)
2925 sub vmstatus {
2926     my ($opt_vmid, $full) = @_;
2927
2928     my $res = {};
2929
2930     my $storecfg = PVE::Storage::config();
2931
2932     my $list = vzlist();
2933     my $defaults = load_defaults();
2934
2935     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2936
2937     my $cpucount = $cpuinfo->{cpus} || 1;
2938
2939     foreach my $vmid (keys %$list) {
2940         next if $opt_vmid && ($vmid ne $opt_vmid);
2941
2942         my $conf = PVE::QemuConfig->load_config($vmid);
2943
2944         my $d = { vmid => $vmid };
2945         $d->{pid} = $list->{$vmid}->{pid};
2946
2947         # fixme: better status?
2948         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2949
2950         my $size = disksize($storecfg, $conf);
2951         if (defined($size)) {
2952             $d->{disk} = 0; # no info available
2953             $d->{maxdisk} = $size;
2954         } else {
2955             $d->{disk} = 0;
2956             $d->{maxdisk} = 0;
2957         }
2958
2959         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2960             * ($conf->{cores} || $defaults->{cores});
2961         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2962         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2963
2964         $d->{name} = $conf->{name} || "VM $vmid";
2965         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2966             : $defaults->{memory}*(1024*1024);
2967
2968         if ($conf->{balloon}) {
2969             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2970             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2971                 : $defaults->{shares};
2972         }
2973
2974         $d->{uptime} = 0;
2975         $d->{cpu} = 0;
2976         $d->{mem} = 0;
2977
2978         $d->{netout} = 0;
2979         $d->{netin} = 0;
2980
2981         $d->{diskread} = 0;
2982         $d->{diskwrite} = 0;
2983
2984         $d->{template} = PVE::QemuConfig->is_template($conf);
2985
2986         $d->{serial} = 1 if conf_has_serial($conf);
2987         $d->{lock} = $conf->{lock} if $conf->{lock};
2988         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2989
2990         $res->{$vmid} = $d;
2991     }
2992
2993     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2994     foreach my $dev (keys %$netdev) {
2995         next if $dev !~ m/^tap([1-9]\d*)i/;
2996         my $vmid = $1;
2997         my $d = $res->{$vmid};
2998         next if !$d;
2999
3000         $d->{netout} += $netdev->{$dev}->{receive};
3001         $d->{netin} += $netdev->{$dev}->{transmit};
3002
3003         if ($full) {
3004             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3005             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3006         }
3007
3008     }
3009
3010     my $ctime = gettimeofday;
3011
3012     foreach my $vmid (keys %$list) {
3013
3014         my $d = $res->{$vmid};
3015         my $pid = $d->{pid};
3016         next if !$pid;
3017
3018         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3019         next if !$pstat; # not running
3020
3021         my $used = $pstat->{utime} + $pstat->{stime};
3022
3023         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3024
3025         if ($pstat->{vsize}) {
3026             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3027         }
3028
3029         my $old = $last_proc_pid_stat->{$pid};
3030         if (!$old) {
3031             $last_proc_pid_stat->{$pid} = {
3032                 time => $ctime,
3033                 used => $used,
3034                 cpu => 0,
3035             };
3036             next;
3037         }
3038
3039         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3040
3041         if ($dtime > 1000) {
3042             my $dutime = $used -  $old->{used};
3043
3044             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3045             $last_proc_pid_stat->{$pid} = {
3046                 time => $ctime,
3047                 used => $used,
3048                 cpu => $d->{cpu},
3049             };
3050         } else {
3051             $d->{cpu} = $old->{cpu};
3052         }
3053     }
3054
3055     return $res if !$full;
3056
3057     my $qmpclient = PVE::QMPClient->new();
3058
3059     my $ballooncb = sub {
3060         my ($vmid, $resp) = @_;
3061
3062         my $info = $resp->{'return'};
3063         return if !$info->{max_mem};
3064
3065         my $d = $res->{$vmid};
3066
3067         # use memory assigned to VM
3068         $d->{maxmem} = $info->{max_mem};
3069         $d->{balloon} = $info->{actual};
3070
3071         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3072             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3073             $d->{freemem} = $info->{free_mem};
3074         }
3075
3076         $d->{ballooninfo} = $info;
3077     };
3078
3079     my $blockstatscb = sub {
3080         my ($vmid, $resp) = @_;
3081         my $data = $resp->{'return'} || [];
3082         my $totalrdbytes = 0;
3083         my $totalwrbytes = 0;
3084
3085         for my $blockstat (@$data) {
3086             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3087             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3088
3089             $blockstat->{device} =~ s/drive-//;
3090             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3091         }
3092         $res->{$vmid}->{diskread} = $totalrdbytes;
3093         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3094     };
3095
3096     my $statuscb = sub {
3097         my ($vmid, $resp) = @_;
3098
3099         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3100         # this fails if ballon driver is not loaded, so this must be
3101         # the last commnand (following command are aborted if this fails).
3102         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3103
3104         my $status = 'unknown';
3105         if (!defined($status = $resp->{'return'}->{status})) {
3106             warn "unable to get VM status\n";
3107             return;
3108         }
3109
3110         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3111     };
3112
3113     foreach my $vmid (keys %$list) {
3114         next if $opt_vmid && ($vmid ne $opt_vmid);
3115         next if !$res->{$vmid}->{pid}; # not running
3116         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3117     }
3118
3119     $qmpclient->queue_execute(undef, 2);
3120
3121     foreach my $vmid (keys %$list) {
3122         next if $opt_vmid && ($vmid ne $opt_vmid);
3123         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3124     }
3125
3126     return $res;
3127 }
3128
3129 sub foreach_drive {
3130     my ($conf, $func, @param) = @_;
3131
3132     foreach my $ds (valid_drive_names()) {
3133         next if !defined($conf->{$ds});
3134
3135         my $drive = parse_drive($ds, $conf->{$ds});
3136         next if !$drive;
3137
3138         &$func($ds, $drive, @param);
3139     }
3140 }
3141
3142 sub foreach_volid {
3143     my ($conf, $func, @param) = @_;
3144
3145     my $volhash = {};
3146
3147     my $test_volid = sub {
3148         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3149
3150         return if !$volid;
3151
3152         $volhash->{$volid}->{cdrom} //= 1;
3153         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3154
3155         $volhash->{$volid}->{replicate} //= 0;
3156         $volhash->{$volid}->{replicate} = 1 if $replicate;
3157
3158         $volhash->{$volid}->{shared} //= 0;
3159         $volhash->{$volid}->{shared} = 1 if $shared;
3160
3161         $volhash->{$volid}->{referenced_in_config} //= 0;
3162         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3163
3164         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3165             if defined($snapname);
3166         $volhash->{$volid}->{size} = $size if $size;
3167     };
3168
3169     foreach_drive($conf, sub {
3170         my ($ds, $drive) = @_;
3171         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3172     });
3173
3174     foreach my $snapname (keys %{$conf->{snapshots}}) {
3175         my $snap = $conf->{snapshots}->{$snapname};
3176         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3177         foreach_drive($snap, sub {
3178             my ($ds, $drive) = @_;
3179             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3180         });
3181     }
3182
3183     foreach my $volid (keys %$volhash) {
3184         &$func($volid, $volhash->{$volid}, @param);
3185     }
3186 }
3187
3188 sub conf_has_serial {
3189     my ($conf) = @_;
3190
3191     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3192         if ($conf->{"serial$i"}) {
3193             return 1;
3194         }
3195     }
3196
3197     return 0;
3198 }
3199
3200 sub conf_has_audio {
3201     my ($conf, $id) = @_;
3202
3203     $id //= 0;
3204     my $audio = $conf->{"audio$id"};
3205     return undef if !defined($audio);
3206
3207     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3208     my $audiodriver = $audioproperties->{driver} // 'spice';
3209
3210     return {
3211         dev => $audioproperties->{device},
3212         dev_id => "audiodev$id",
3213         backend => $audiodriver,
3214         backend_id => "$audiodriver-backend${id}",
3215     };
3216 }
3217
3218 sub vga_conf_has_spice {
3219     my ($vga) = @_;
3220
3221     my $vgaconf = parse_vga($vga);
3222     my $vgatype = $vgaconf->{type};
3223     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3224
3225     return $1 || 1;
3226 }
3227
3228 sub is_native($) {
3229     my ($arch) = @_;
3230     return get_host_arch() eq $arch;
3231 }
3232
3233 sub get_vm_arch {
3234     my ($conf) = @_;
3235     return $conf->{arch} // get_host_arch();
3236 }
3237
3238 my $default_machines = {
3239     x86_64 => 'pc',
3240     aarch64 => 'virt',
3241 };
3242
3243 sub get_vm_machine {
3244     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3245
3246     my $machine = $forcemachine || $conf->{machine};
3247
3248     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3249         $arch //= 'x86_64';
3250         $machine ||= $default_machines->{$arch};
3251         if ($add_pve_version) {
3252             $kvmversion //= kvm_user_version();
3253             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3254             $machine .= "+pve$pvever";
3255         }
3256     }
3257
3258     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
3259         # for version-pinned machines that do not include a pve-version (e.g.
3260         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3261         $machine .= '+pve0';
3262     }
3263
3264     return $machine;
3265 }
3266
3267 sub get_ovmf_files($) {
3268     my ($arch) = @_;
3269
3270     my $ovmf = $OVMF->{$arch}
3271         or die "no OVMF images known for architecture '$arch'\n";
3272
3273     return @$ovmf;
3274 }
3275
3276 my $Arch2Qemu = {
3277     aarch64 => '/usr/bin/qemu-system-aarch64',
3278     x86_64 => '/usr/bin/qemu-system-x86_64',
3279 };
3280 sub get_command_for_arch($) {
3281     my ($arch) = @_;
3282     return '/usr/bin/kvm' if is_native($arch);
3283
3284     my $cmd = $Arch2Qemu->{$arch}
3285         or die "don't know how to emulate architecture '$arch'\n";
3286     return $cmd;
3287 }
3288
3289 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3290 # to use in a QEMU command line (-cpu element), first array_intersect the result
3291 # of query_supported_ with query_understood_. This is necessary because:
3292 #
3293 # a) query_understood_ returns flags the host cannot use and
3294 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3295 #    flags, but CPU settings - with most of them being flags. Those settings
3296 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3297 #
3298 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3299 # expensive. If you need the value returned from this, you can get it much
3300 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3301 # $accel being 'kvm' or 'tcg'.
3302 #
3303 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3304 # changes, automatically populating pmxcfs.
3305 #
3306 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3307 # since kvm and tcg machines support different flags
3308 #
3309 sub query_supported_cpu_flags {
3310     my ($arch) = @_;
3311
3312     $arch //= get_host_arch();
3313     my $default_machine = $default_machines->{$arch};
3314
3315     my $flags = {};
3316
3317     # FIXME: Once this is merged, the code below should work for ARM as well:
3318     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3319     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3320         $arch eq "aarch64";
3321
3322     my $kvm_supported = defined(kvm_version());
3323     my $qemu_cmd = get_command_for_arch($arch);
3324     my $fakevmid = -1;
3325     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3326
3327     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3328     my $query_supported_run_qemu = sub {
3329         my ($kvm) = @_;
3330
3331         my $flags = {};
3332         my $cmd = [
3333             $qemu_cmd,
3334             '-machine', $default_machine,
3335             '-display', 'none',
3336             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3337             '-mon', 'chardev=qmp,mode=control',
3338             '-pidfile', $pidfile,
3339             '-S', '-daemonize'
3340         ];
3341
3342         if (!$kvm) {
3343             push @$cmd, '-accel', 'tcg';
3344         }
3345
3346         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3347         die "QEMU flag querying VM exited with code " . $rc if $rc;
3348
3349         eval {
3350             my $cmd_result = mon_cmd(
3351                 $fakevmid,
3352                 'query-cpu-model-expansion',
3353                 type => 'full',
3354                 model => { name => 'host' }
3355             );
3356
3357             my $props = $cmd_result->{model}->{props};
3358             foreach my $prop (keys %$props) {
3359                 next if $props->{$prop} ne '1';
3360                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3361                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3362                 # We only keep those with underscores, to match /proc/cpuinfo
3363                 $prop =~ s/\.|-/_/g;
3364                 $flags->{$prop} = 1;
3365             }
3366         };
3367         my $err = $@;
3368
3369         # force stop with 10 sec timeout and 'nocheck'
3370         # always stop, even if QMP failed
3371         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3372
3373         die $err if $err;
3374
3375         return [ sort keys %$flags ];
3376     };
3377
3378     # We need to query QEMU twice, since KVM and TCG have different supported flags
3379     PVE::QemuConfig->lock_config($fakevmid, sub {
3380         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3381         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3382
3383         if ($kvm_supported) {
3384             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3385             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3386         }
3387     });
3388
3389     return $flags;
3390 }
3391
3392 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3393 my $understood_cpu_flag_dir = "/usr/share/kvm";
3394 sub query_understood_cpu_flags {
3395     my $arch = get_host_arch();
3396     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3397
3398     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3399         if ! -e $filepath;
3400
3401     my $raw = file_get_contents($filepath);
3402     $raw =~ s/^\s+|\s+$//g;
3403     my @flags = split(/\s+/, $raw);
3404
3405     return \@flags;
3406 }
3407
3408 sub config_to_command {
3409     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3410
3411     my $cmd = [];
3412     my $globalFlags = [];
3413     my $machineFlags = [];
3414     my $rtcFlags = [];
3415     my $devices = [];
3416     my $pciaddr = '';
3417     my $bridges = {};
3418     my $ostype = $conf->{ostype};
3419     my $winversion = windows_version($ostype);
3420     my $kvm = $conf->{kvm};
3421     my $nodename = nodename();
3422
3423     my $arch = get_vm_arch($conf);
3424     my $kvm_binary = get_command_for_arch($arch);
3425     my $kvmver = kvm_user_version($kvm_binary);
3426
3427     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3428         $kvmver //= "undefined";
3429         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3430     }
3431
3432     my $add_pve_version = min_version($kvmver, 4, 1);
3433
3434     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3435     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3436     $kvm //= 1 if is_native($arch);
3437
3438     $machine_version =~ m/(\d+)\.(\d+)/;
3439     my ($machine_major, $machine_minor) = ($1, $2);
3440     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3441         if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);
3442
3443     if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3444         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3445         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
3446           . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
3447     }
3448
3449     # if a specific +pve version is required for a feature, use $version_guard
3450     # instead of min_version to allow machines to be run with the minimum
3451     # required version
3452     my $required_pve_version = 0;
3453     my $version_guard = sub {
3454         my ($major, $minor, $pve) = @_;
3455         return 0 if !min_version($machine_version, $major, $minor, $pve);
3456         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3457         return 1;
3458     };
3459
3460     if ($kvm) {
3461         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3462             if !defined kvm_version();
3463     }
3464
3465     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3466     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3467     my $use_old_bios_files = undef;
3468     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3469
3470     my $cpuunits = defined($conf->{cpuunits}) ?
3471             $conf->{cpuunits} : $defaults->{cpuunits};
3472
3473     push @$cmd, $kvm_binary;
3474
3475     push @$cmd, '-id', $vmid;
3476
3477     my $vmname = $conf->{name} || "vm$vmid";
3478
3479     push @$cmd, '-name', $vmname;
3480
3481     my $use_virtio = 0;
3482
3483     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3484     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3485     push @$cmd, '-mon', "chardev=qmp,mode=control";
3486
3487     if (min_version($machine_version, 2, 12)) {
3488         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3489         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3490     }
3491
3492     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3493
3494     push @$cmd, '-daemonize';
3495
3496     if ($conf->{smbios1}) {
3497         my $smbios_conf = parse_smbios1($conf->{smbios1});
3498         if ($smbios_conf->{base64}) {
3499             # Do not pass base64 flag to qemu
3500             delete $smbios_conf->{base64};
3501             my $smbios_string = "";
3502             foreach my $key (keys %$smbios_conf) {
3503                 my $value;
3504                 if ($key eq "uuid") {
3505                     $value = $smbios_conf->{uuid}
3506                 } else {
3507                     $value = decode_base64($smbios_conf->{$key});
3508                 }
3509                 # qemu accepts any binary data, only commas need escaping by double comma
3510                 $value =~ s/,/,,/g;
3511                 $smbios_string .= "," . $key . "=" . $value if $value;
3512             }
3513             push @$cmd, '-smbios', "type=1" . $smbios_string;
3514         } else {
3515             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3516         }
3517     }
3518
3519     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3520     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3521         die "uefi base image not found\n" if ! -f $ovmf_code;
3522
3523         my $path;
3524         my $format;
3525         if (my $efidisk = $conf->{efidisk0}) {
3526             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3527             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3528             $format = $d->{format};
3529             if ($storeid) {
3530                 $path = PVE::Storage::path($storecfg, $d->{file});
3531                 if (!defined($format)) {
3532                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3533                     $format = qemu_img_format($scfg, $volname);
3534                 }
3535             } else {
3536                 $path = $d->{file};
3537                 die "efidisk format must be specified\n"
3538                     if !defined($format);
3539             }
3540         } else {
3541             warn "no efidisk configured! Using temporary efivars disk.\n";
3542             $path = "/tmp/$vmid-ovmf.fd";
3543             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3544             $format = 'raw';
3545         }
3546
3547         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3548         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3549     }
3550
3551     # load q35 config
3552     if ($q35) {
3553         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3554         if (min_version($machine_version, 4, 0)) {
3555             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3556         } else {
3557             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3558         }
3559     }
3560
3561     if ($conf->{vmgenid}) {
3562         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3563     }
3564
3565     # add usb controllers
3566     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3567     push @$devices, @usbcontrollers if @usbcontrollers;
3568     my $vga = parse_vga($conf->{vga});
3569
3570     my $qxlnum = vga_conf_has_spice($conf->{vga});
3571     $vga->{type} = 'qxl' if $qxlnum;
3572
3573     if (!$vga->{type}) {
3574         if ($arch eq 'aarch64') {
3575             $vga->{type} = 'virtio';
3576         } elsif (min_version($machine_version, 2, 9)) {
3577             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3578         } else {
3579             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3580         }
3581     }
3582
3583     # enable absolute mouse coordinates (needed by vnc)
3584     my $tablet;
3585     if (defined($conf->{tablet})) {
3586         $tablet = $conf->{tablet};
3587     } else {
3588         $tablet = $defaults->{tablet};
3589         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3590         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3591     }
3592
3593     if ($tablet) {
3594         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3595         my $kbd = print_keyboarddevice_full($conf, $arch);
3596         push @$devices, '-device', $kbd if defined($kbd);
3597     }
3598
3599     my $kvm_off = 0;
3600     my $gpu_passthrough;
3601
3602     # host pci devices
3603     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3604         my $id = "hostpci$i";
3605         my $d = parse_hostpci($conf->{$id});
3606         next if !$d;
3607
3608         if (my $pcie = $d->{pcie}) {
3609             die "q35 machine model is not enabled" if !$q35;
3610             # win7 wants to have the pcie devices directly on the pcie bus
3611             # instead of in the root port
3612             if ($winversion == 7) {
3613                 $pciaddr = print_pcie_addr("${id}bus0");
3614             } else {
3615                 # add more root ports if needed, 4 are present by default
3616                 # by pve-q35 cfgs, rest added here on demand.
3617                 if ($i > 3) {
3618                     push @$devices, '-device', print_pcie_root_port($i);
3619                 }
3620                 $pciaddr = print_pcie_addr($id);
3621             }
3622         } else {
3623             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3624         }
3625
3626         my $xvga = '';
3627         if ($d->{'x-vga'}) {
3628             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3629             $kvm_off = 1;
3630             $vga->{type} = 'none' if !defined($conf->{vga});
3631             $gpu_passthrough = 1;
3632         }
3633
3634         my $pcidevices = $d->{pciid};
3635         my $multifunction = 1 if @$pcidevices > 1;
3636
3637         my $sysfspath;
3638         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3639             my $pci_id = $pcidevices->[0]->{id};
3640             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3641             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3642         } elsif ($d->{mdev}) {
3643             warn "ignoring mediated device '$id' with multifunction device\n";
3644         }
3645
3646         my $j=0;
3647         foreach my $pcidevice (@$pcidevices) {
3648             my $devicestr = "vfio-pci";
3649
3650             if ($sysfspath) {
3651                 $devicestr .= ",sysfsdev=$sysfspath";
3652             } else {
3653                 $devicestr .= ",host=$pcidevice->{id}";
3654             }
3655
3656             my $mf_addr = $multifunction ? ".$j" : '';
3657             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3658
3659             if ($j == 0) {
3660                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3661                 $devicestr .= "$xvga";
3662                 $devicestr .= ",multifunction=on" if $multifunction;
3663                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3664             }
3665
3666             push @$devices, '-device', $devicestr;
3667             $j++;
3668         }
3669     }
3670
3671     # usb devices
3672     my $usb_dev_features = {};
3673     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3674
3675     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3676     push @$devices, @usbdevices if @usbdevices;
3677     # serial devices
3678     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3679         if (my $path = $conf->{"serial$i"}) {
3680             if ($path eq 'socket') {
3681                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3682                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3683                 # On aarch64, serial0 is the UART device. Qemu only allows
3684                 # connecting UART devices via the '-serial' command line, as
3685                 # the device has a fixed slot on the hardware...
3686                 if ($arch eq 'aarch64' && $i == 0) {
3687                     push @$devices, '-serial', "chardev:serial$i";
3688                 } else {
3689                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3690                 }
3691             } else {
3692                 die "no such serial device\n" if ! -c $path;
3693                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3694                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3695             }
3696         }
3697     }
3698
3699     # parallel devices
3700     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3701         if (my $path = $conf->{"parallel$i"}) {
3702             die "no such parallel device\n" if ! -c $path;
3703             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3704             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3705             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3706         }
3707     }
3708
3709     if (my $audio = conf_has_audio($conf)) {
3710
3711         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3712
3713         my $id = $audio->{dev_id};
3714         if ($audio->{dev} eq 'AC97') {
3715             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3716         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3717             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3718             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3719             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3720         } else {
3721             die "unkown audio device '$audio->{dev}', implement me!";
3722         }
3723
3724         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3725     }
3726
3727     my $sockets = 1;
3728     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3729     $sockets = $conf->{sockets} if  $conf->{sockets};
3730
3731     my $cores = $conf->{cores} || 1;
3732
3733     my $maxcpus = $sockets * $cores;
3734
3735     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3736
3737     my $allowed_vcpus = $cpuinfo->{cpus};
3738
3739     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3740         if ($allowed_vcpus < $maxcpus);
3741
3742     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3743
3744         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3745         for (my $i = 2; $i <= $vcpus; $i++)  {
3746             my $cpustr = print_cpu_device($conf,$i);
3747             push @$cmd, '-device', $cpustr;
3748         }
3749
3750     } else {
3751
3752         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753     }
3754     push @$cmd, '-nodefaults';
3755
3756     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3757
3758     my $bootindex_hash = {};
3759     my $i = 1;
3760     foreach my $o (split(//, $bootorder)) {
3761         $bootindex_hash->{$o} = $i*100;
3762         $i++;
3763     }
3764
3765     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3766
3767     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3768
3769     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3770
3771     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3772         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3773         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3774         push @$cmd,  '-vnc', "unix:$socket,password";
3775     } else {
3776         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3777         push @$cmd, '-nographic';
3778     }
3779
3780     # time drift fix
3781     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3782
3783     my $useLocaltime = $conf->{localtime};
3784
3785     if ($winversion >= 5) { # windows
3786         $useLocaltime = 1 if !defined($conf->{localtime});
3787
3788         # use time drift fix when acpi is enabled
3789         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3790             $tdf = 1 if !defined($conf->{tdf});
3791         }
3792     }
3793
3794     if ($winversion >= 6) {
3795         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3796         push @$cmd, '-no-hpet';
3797     }
3798
3799     push @$rtcFlags, 'driftfix=slew' if $tdf;
3800
3801     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3802         push @$rtcFlags, "base=$conf->{startdate}";
3803     } elsif ($useLocaltime) {
3804         push @$rtcFlags, 'base=localtime';
3805     }
3806
3807     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3808
3809     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3810
3811     push @$cmd, '-S' if $conf->{freeze};
3812
3813     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3814
3815     my $guest_agent = parse_guest_agent($conf);
3816
3817     if ($guest_agent->{enabled}) {
3818         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3819         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3820
3821         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3822             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3823             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3824             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3825         } elsif ($guest_agent->{type} eq 'isa') {
3826             push @$devices, '-device', "isa-serial,chardev=qga0";
3827         }
3828     }
3829
3830     my $spice_port;
3831
3832     if ($qxlnum) {
3833         if ($qxlnum > 1) {
3834             if ($winversion){
3835                 for(my $i = 1; $i < $qxlnum; $i++){
3836                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3837                 }
3838             } else {
3839                 # assume other OS works like Linux
3840                 my ($ram, $vram) = ("134217728", "67108864");
3841                 if ($vga->{memory}) {
3842                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3843                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3844                 }
3845                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3846                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3847             }
3848         }
3849
3850         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3851
3852         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3853         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3854         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3855
3856         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3857         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3858         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3859
3860         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3861         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3862
3863         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3864         if ($spice_enhancement->{foldersharing}) {
3865             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3866             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3867         }
3868
3869         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3870         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3871         push @$devices, '-spice', "$spice_opts";
3872     }
3873
3874     # enable balloon by default, unless explicitly disabled
3875     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3876         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3877         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3878     }
3879
3880     if ($conf->{watchdog}) {
3881         my $wdopts = parse_watchdog($conf->{watchdog});
3882         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3883         my $watchdog = $wdopts->{model} || 'i6300esb';
3884         push @$devices, '-device', "$watchdog$pciaddr";
3885         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3886     }
3887
3888     my $vollist = [];
3889     my $scsicontroller = {};
3890     my $ahcicontroller = {};
3891     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3892
3893     # Add iscsi initiator name if available
3894     if (my $initiator = get_initiator_name()) {
3895         push @$devices, '-iscsi', "initiator-name=$initiator";
3896     }
3897
3898     foreach_drive($conf, sub {
3899         my ($ds, $drive) = @_;
3900
3901         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3902             push @$vollist, $drive->{file};
3903         }
3904
3905         # ignore efidisk here, already added in bios/fw handling code above
3906         return if $drive->{interface} eq 'efidisk';
3907
3908         $use_virtio = 1 if $ds =~ m/^virtio/;
3909
3910         if (drive_is_cdrom ($drive)) {
3911             if ($bootindex_hash->{d}) {
3912                 $drive->{bootindex} = $bootindex_hash->{d};
3913                 $bootindex_hash->{d} += 1;
3914             }
3915         } else {
3916             if ($bootindex_hash->{c}) {
3917                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3918                 $bootindex_hash->{c} += 1;
3919             }
3920         }
3921
3922         if($drive->{interface} eq 'virtio'){
3923            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3924         }
3925
3926         if ($drive->{interface} eq 'scsi') {
3927
3928             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3929
3930             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3931                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3932
3933             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3934             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3935
3936             my $iothread = '';
3937             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3938                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3939                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3940             } elsif ($drive->{iothread}) {
3941                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3942             }
3943
3944             my $queues = '';
3945             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3946                 $queues = ",num_queues=$drive->{queues}";
3947             }
3948
3949             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3950             $scsicontroller->{$controller}=1;
3951         }
3952
3953         if ($drive->{interface} eq 'sata') {
3954            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3955            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3956            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3957            $ahcicontroller->{$controller}=1;
3958         }
3959
3960         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3961         push @$devices, '-drive',$drive_cmd;
3962         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3963     });
3964
3965     for (my $i = 0; $i < $MAX_NETS; $i++) {
3966          next if !$conf->{"net$i"};
3967          my $d = parse_net($conf->{"net$i"});
3968          next if !$d;
3969
3970          $use_virtio = 1 if $d->{model} eq 'virtio';
3971
3972          if ($bootindex_hash->{n}) {
3973             $d->{bootindex} = $bootindex_hash->{n};
3974             $bootindex_hash->{n} += 1;
3975          }
3976
3977          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3978          push @$devices, '-netdev', $netdevfull;
3979
3980          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3981          push @$devices, '-device', $netdevicefull;
3982     }
3983
3984     if ($conf->{ivshmem}) {
3985         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3986
3987         my $bus;
3988         if ($q35) {
3989             $bus = print_pcie_addr("ivshmem");
3990         } else {
3991             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3992         }
3993
3994         my $ivshmem_name = $ivshmem->{name} // $vmid;
3995         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3996
3997         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3998         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3999     }
4000
4001     # pci.4 is nested in pci.1
4002     $bridges->{1} = 1 if $bridges->{4};
4003
4004     if (!$q35) {
4005         # add pci bridges
4006         if (min_version($machine_version, 2, 3)) {
4007            $bridges->{1} = 1;
4008            $bridges->{2} = 1;
4009         }
4010
4011         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4012
4013     }
4014
4015     for my $k (sort {$b cmp $a} keys %$bridges) {
4016         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4017         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4018         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4019         if ($q35) {
4020             # add after -readconfig pve-q35.cfg
4021             splice @$devices, 2, 0, '-device', $devstr;
4022         } else {
4023             unshift @$devices, '-device', $devstr if $k > 0;
4024         }
4025     }
4026
4027     if (!$kvm) {
4028         push @$machineFlags, 'accel=tcg';
4029     }
4030
4031     my $machine_type_min = $machine_type;
4032     if ($add_pve_version) {
4033         $machine_type_min =~ s/\+pve\d+$//;
4034         $machine_type_min .= "+pve$required_pve_version";
4035     }
4036     push @$machineFlags, "type=${machine_type_min}";
4037
4038     push @$cmd, @$devices;
4039     push @$cmd, '-rtc', join(',', @$rtcFlags)
4040         if scalar(@$rtcFlags);
4041     push @$cmd, '-machine', join(',', @$machineFlags)
4042         if scalar(@$machineFlags);
4043     push @$cmd, '-global', join(',', @$globalFlags)
4044         if scalar(@$globalFlags);
4045
4046     if (my $vmstate = $conf->{vmstate}) {
4047         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4048         push @$vollist, $vmstate;
4049         push @$cmd, '-loadstate', $statepath;
4050         print "activating and using '$vmstate' as vmstate\n";
4051     }
4052
4053     # add custom args
4054     if ($conf->{args}) {
4055         my $aa = PVE::Tools::split_args($conf->{args});
4056         push @$cmd, @$aa;
4057     }
4058
4059     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4060 }
4061
4062 sub spice_port {
4063     my ($vmid) = @_;
4064
4065     my $res = mon_cmd($vmid, 'query-spice');
4066
4067     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4068 }
4069
4070 sub vm_devices_list {
4071     my ($vmid) = @_;
4072
4073     my $res = mon_cmd($vmid, 'query-pci');
4074     my $devices_to_check = [];
4075     my $devices = {};
4076     foreach my $pcibus (@$res) {
4077         push @$devices_to_check, @{$pcibus->{devices}},
4078     }
4079
4080     while (@$devices_to_check) {
4081         my $to_check = [];
4082         for my $d (@$devices_to_check) {
4083             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4084             next if !$d->{'pci_bridge'};
4085
4086             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4087             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4088         }
4089         $devices_to_check = $to_check;
4090     }
4091
4092     my $resblock = mon_cmd($vmid, 'query-block');
4093     foreach my $block (@$resblock) {
4094         if($block->{device} =~ m/^drive-(\S+)/){
4095                 $devices->{$1} = 1;
4096         }
4097     }
4098
4099     my $resmice = mon_cmd($vmid, 'query-mice');
4100     foreach my $mice (@$resmice) {
4101         if ($mice->{name} eq 'QEMU HID Tablet') {
4102             $devices->{tablet} = 1;
4103             last;
4104         }
4105     }
4106
4107     # for usb devices there is no query-usb
4108     # but we can iterate over the entries in
4109     # qom-list path=/machine/peripheral
4110     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4111     foreach my $per (@$resperipheral) {
4112         if ($per->{name} =~ m/^usb\d+$/) {
4113             $devices->{$per->{name}} = 1;
4114         }
4115     }
4116
4117     return $devices;
4118 }
4119
4120 sub vm_deviceplug {
4121     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4122
4123     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4124
4125     my $devices_list = vm_devices_list($vmid);
4126     return 1 if defined($devices_list->{$deviceid});
4127
4128     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4129
4130     if ($deviceid eq 'tablet') {
4131
4132         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4133
4134     } elsif ($deviceid eq 'keyboard') {
4135
4136         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4137
4138     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4139
4140         die "usb hotplug currently not reliable\n";
4141         # since we can't reliably hot unplug all added usb devices
4142         # and usb passthrough disables live migration
4143         # we disable usb hotplugging for now
4144         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4145
4146     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4147
4148         qemu_iothread_add($vmid, $deviceid, $device);
4149
4150         qemu_driveadd($storecfg, $vmid, $device);
4151         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4152
4153         qemu_deviceadd($vmid, $devicefull);
4154         eval { qemu_deviceaddverify($vmid, $deviceid); };
4155         if (my $err = $@) {
4156             eval { qemu_drivedel($vmid, $deviceid); };
4157             warn $@ if $@;
4158             die $err;
4159         }
4160
4161     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4162
4163
4164         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4165         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4166         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4167
4168         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4169
4170         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4171             qemu_iothread_add($vmid, $deviceid, $device);
4172             $devicefull .= ",iothread=iothread-$deviceid";
4173         }
4174
4175         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4176             $devicefull .= ",num_queues=$device->{queues}";
4177         }
4178
4179         qemu_deviceadd($vmid, $devicefull);
4180         qemu_deviceaddverify($vmid, $deviceid);
4181
4182     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4183
4184         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4185         qemu_driveadd($storecfg, $vmid, $device);
4186
4187         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4188         eval { qemu_deviceadd($vmid, $devicefull); };
4189         if (my $err = $@) {
4190             eval { qemu_drivedel($vmid, $deviceid); };
4191             warn $@ if $@;
4192             die $err;
4193         }
4194
4195     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4196
4197         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4198
4199         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4200         my $use_old_bios_files = undef;
4201         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4202
4203         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4204         qemu_deviceadd($vmid, $netdevicefull);
4205         eval {
4206             qemu_deviceaddverify($vmid, $deviceid);
4207             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4208         };
4209         if (my $err = $@) {
4210             eval { qemu_netdevdel($vmid, $deviceid); };
4211             warn $@ if $@;
4212             die $err;
4213         }
4214
4215     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4216
4217         my $bridgeid = $2;
4218         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4219         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4220
4221         qemu_deviceadd($vmid, $devicefull);
4222         qemu_deviceaddverify($vmid, $deviceid);
4223
4224     } else {
4225         die "can't hotplug device '$deviceid'\n";
4226     }
4227
4228     return 1;
4229 }
4230
4231 # fixme: this should raise exceptions on error!
4232 sub vm_deviceunplug {
4233     my ($vmid, $conf, $deviceid) = @_;
4234
4235     my $devices_list = vm_devices_list($vmid);
4236     return 1 if !defined($devices_list->{$deviceid});
4237
4238     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4239
4240     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4241
4242         qemu_devicedel($vmid, $deviceid);
4243
4244     } elsif ($deviceid =~ m/^usb\d+$/) {
4245
4246         die "usb hotplug currently not reliable\n";
4247         # when unplugging usb devices this way,
4248         # there may be remaining usb controllers/hubs
4249         # so we disable it for now
4250         qemu_devicedel($vmid, $deviceid);
4251         qemu_devicedelverify($vmid, $deviceid);
4252
4253     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4254
4255         qemu_devicedel($vmid, $deviceid);
4256         qemu_devicedelverify($vmid, $deviceid);
4257         qemu_drivedel($vmid, $deviceid);
4258         qemu_iothread_del($conf, $vmid, $deviceid);
4259
4260     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4261
4262         qemu_devicedel($vmid, $deviceid);
4263         qemu_devicedelverify($vmid, $deviceid);
4264         qemu_iothread_del($conf, $vmid, $deviceid);
4265
4266     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4267
4268         qemu_devicedel($vmid, $deviceid);
4269         qemu_drivedel($vmid, $deviceid);
4270         qemu_deletescsihw($conf, $vmid, $deviceid);
4271
4272     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4273
4274         qemu_devicedel($vmid, $deviceid);
4275         qemu_devicedelverify($vmid, $deviceid);
4276         qemu_netdevdel($vmid, $deviceid);
4277
4278     } else {
4279         die "can't unplug device '$deviceid'\n";
4280     }
4281
4282     return 1;
4283 }
4284
4285 sub qemu_deviceadd {
4286     my ($vmid, $devicefull) = @_;
4287
4288     $devicefull = "driver=".$devicefull;
4289     my %options =  split(/[=,]/, $devicefull);
4290
4291     mon_cmd($vmid, "device_add" , %options);
4292 }
4293
4294 sub qemu_devicedel {
4295     my ($vmid, $deviceid) = @_;
4296
4297     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4298 }
4299
4300 sub qemu_iothread_add {
4301     my($vmid, $deviceid, $device) = @_;
4302
4303     if ($device->{iothread}) {
4304         my $iothreads = vm_iothreads_list($vmid);
4305         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4306     }
4307 }
4308
4309 sub qemu_iothread_del {
4310     my($conf, $vmid, $deviceid) = @_;
4311
4312     my $confid = $deviceid;
4313     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4314         $confid = 'scsi' . $1;
4315     }
4316     my $device = parse_drive($confid, $conf->{$confid});
4317     if ($device->{iothread}) {
4318         my $iothreads = vm_iothreads_list($vmid);
4319         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4320     }
4321 }
4322
4323 sub qemu_objectadd {
4324     my($vmid, $objectid, $qomtype) = @_;
4325
4326     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4327
4328     return 1;
4329 }
4330
4331 sub qemu_objectdel {
4332     my($vmid, $objectid) = @_;
4333
4334     mon_cmd($vmid, "object-del", id => $objectid);
4335
4336     return 1;
4337 }
4338
4339 sub qemu_driveadd {
4340     my ($storecfg, $vmid, $device) = @_;
4341
4342     my $drive = print_drive_full($storecfg, $vmid, $device);
4343     $drive =~ s/\\/\\\\/g;
4344     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4345
4346     # If the command succeeds qemu prints: "OK"
4347     return 1 if $ret =~ m/OK/s;
4348
4349     die "adding drive failed: $ret\n";
4350 }
4351
4352 sub qemu_drivedel {
4353     my($vmid, $deviceid) = @_;
4354
4355     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4356     $ret =~ s/^\s+//;
4357
4358     return 1 if $ret eq "";
4359
4360     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4361     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4362
4363     die "deleting drive $deviceid failed : $ret\n";
4364 }
4365
4366 sub qemu_deviceaddverify {
4367     my ($vmid, $deviceid) = @_;
4368
4369     for (my $i = 0; $i <= 5; $i++) {
4370          my $devices_list = vm_devices_list($vmid);
4371          return 1 if defined($devices_list->{$deviceid});
4372          sleep 1;
4373     }
4374
4375     die "error on hotplug device '$deviceid'\n";
4376 }
4377
4378
4379 sub qemu_devicedelverify {
4380     my ($vmid, $deviceid) = @_;
4381
4382     # need to verify that the device is correctly removed as device_del
4383     # is async and empty return is not reliable
4384
4385     for (my $i = 0; $i <= 5; $i++) {
4386          my $devices_list = vm_devices_list($vmid);
4387          return 1 if !defined($devices_list->{$deviceid});
4388          sleep 1;
4389     }
4390
4391     die "error on hot-unplugging device '$deviceid'\n";
4392 }
4393
4394 sub qemu_findorcreatescsihw {
4395     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4396
4397     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4398
4399     my $scsihwid="$controller_prefix$controller";
4400     my $devices_list = vm_devices_list($vmid);
4401
4402     if(!defined($devices_list->{$scsihwid})) {
4403         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4404     }
4405
4406     return 1;
4407 }
4408
4409 sub qemu_deletescsihw {
4410     my ($conf, $vmid, $opt) = @_;
4411
4412     my $device = parse_drive($opt, $conf->{$opt});
4413
4414     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4415         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4416         return 1;
4417     }
4418
4419     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4420
4421     my $devices_list = vm_devices_list($vmid);
4422     foreach my $opt (keys %{$devices_list}) {
4423         if (PVE::QemuServer::is_valid_drivename($opt)) {
4424             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4425             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4426                 return 1;
4427             }
4428         }
4429     }
4430
4431     my $scsihwid="scsihw$controller";
4432
4433     vm_deviceunplug($vmid, $conf, $scsihwid);
4434
4435     return 1;
4436 }
4437
4438 sub qemu_add_pci_bridge {
4439     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4440
4441     my $bridges = {};
4442
4443     my $bridgeid;
4444
4445     print_pci_addr($device, $bridges, $arch, $machine_type);
4446
4447     while (my ($k, $v) = each %$bridges) {
4448         $bridgeid = $k;
4449     }
4450     return 1 if !defined($bridgeid) || $bridgeid < 1;
4451
4452     my $bridge = "pci.$bridgeid";
4453     my $devices_list = vm_devices_list($vmid);
4454
4455     if (!defined($devices_list->{$bridge})) {
4456         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4457     }
4458
4459     return 1;
4460 }
4461
4462 sub qemu_set_link_status {
4463     my ($vmid, $device, $up) = @_;
4464
4465     mon_cmd($vmid, "set_link", name => $device,
4466                up => $up ? JSON::true : JSON::false);
4467 }
4468
4469 sub qemu_netdevadd {
4470     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4471
4472     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4473     my %options =  split(/[=,]/, $netdev);
4474
4475     mon_cmd($vmid, "netdev_add",  %options);
4476     return 1;
4477 }
4478
4479 sub qemu_netdevdel {
4480     my ($vmid, $deviceid) = @_;
4481
4482     mon_cmd($vmid, "netdev_del", id => $deviceid);
4483 }
4484
4485 sub qemu_usb_hotplug {
4486     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4487
4488     return if !$device;
4489
4490     # remove the old one first
4491     vm_deviceunplug($vmid, $conf, $deviceid);
4492
4493     # check if xhci controller is necessary and available
4494     if ($device->{usb3}) {
4495
4496         my $devicelist = vm_devices_list($vmid);
4497
4498         if (!$devicelist->{xhci}) {
4499             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4500             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4501         }
4502     }
4503     my $d = parse_usb_device($device->{host});
4504     $d->{usb3} = $device->{usb3};
4505
4506     # add the new one
4507     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4508 }
4509
4510 sub qemu_cpu_hotplug {
4511     my ($vmid, $conf, $vcpus) = @_;
4512
4513     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4514
4515     my $sockets = 1;
4516     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4517     $sockets = $conf->{sockets} if  $conf->{sockets};
4518     my $cores = $conf->{cores} || 1;
4519     my $maxcpus = $sockets * $cores;
4520
4521     $vcpus = $maxcpus if !$vcpus;
4522
4523     die "you can't add more vcpus than maxcpus\n"
4524         if $vcpus > $maxcpus;
4525
4526     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4527
4528     if ($vcpus < $currentvcpus) {
4529
4530         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4531
4532             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4533                 qemu_devicedel($vmid, "cpu$i");
4534                 my $retry = 0;
4535                 my $currentrunningvcpus = undef;
4536                 while (1) {
4537                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4538                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4539                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4540                     $retry++;
4541                     sleep 1;
4542                 }
4543                 #update conf after each succesfull cpu unplug
4544                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4545                 PVE::QemuConfig->write_config($vmid, $conf);
4546             }
4547         } else {
4548             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4549         }
4550
4551         return;
4552     }
4553
4554     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4555     die "vcpus in running vm does not match its configuration\n"
4556         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4557
4558     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4559
4560         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4561             my $cpustr = print_cpu_device($conf, $i);
4562             qemu_deviceadd($vmid, $cpustr);
4563
4564             my $retry = 0;
4565             my $currentrunningvcpus = undef;
4566             while (1) {
4567                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4568                 last if scalar(@{$currentrunningvcpus}) == $i;
4569                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4570                 sleep 1;
4571                 $retry++;
4572             }
4573             #update conf after each succesfull cpu hotplug
4574             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4575             PVE::QemuConfig->write_config($vmid, $conf);
4576         }
4577     } else {
4578
4579         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4580             mon_cmd($vmid, "cpu-add", id => int($i));
4581         }
4582     }
4583 }
4584
4585 sub qemu_block_set_io_throttle {
4586     my ($vmid, $deviceid,
4587         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4588         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4589         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4590         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4591
4592     return if !check_running($vmid) ;
4593
4594     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4595         bps => int($bps),
4596         bps_rd => int($bps_rd),
4597         bps_wr => int($bps_wr),
4598         iops => int($iops),
4599         iops_rd => int($iops_rd),
4600         iops_wr => int($iops_wr),
4601         bps_max => int($bps_max),
4602         bps_rd_max => int($bps_rd_max),
4603         bps_wr_max => int($bps_wr_max),
4604         iops_max => int($iops_max),
4605         iops_rd_max => int($iops_rd_max),
4606         iops_wr_max => int($iops_wr_max),
4607         bps_max_length => int($bps_max_length),
4608         bps_rd_max_length => int($bps_rd_max_length),
4609         bps_wr_max_length => int($bps_wr_max_length),
4610         iops_max_length => int($iops_max_length),
4611         iops_rd_max_length => int($iops_rd_max_length),
4612         iops_wr_max_length => int($iops_wr_max_length),
4613     );
4614
4615 }
4616
4617 # old code, only used to shutdown old VM after update
4618 sub __read_avail {
4619     my ($fh, $timeout) = @_;
4620
4621     my $sel = new IO::Select;
4622     $sel->add($fh);
4623
4624     my $res = '';
4625     my $buf;
4626
4627     my @ready;
4628     while (scalar (@ready = $sel->can_read($timeout))) {
4629         my $count;
4630         if ($count = $fh->sysread($buf, 8192)) {
4631             if ($buf =~ /^(.*)\(qemu\) $/s) {
4632                 $res .= $1;
4633                 last;
4634             } else {
4635                 $res .= $buf;
4636             }
4637         } else {
4638             if (!defined($count)) {
4639                 die "$!\n";
4640             }
4641             last;
4642         }
4643     }
4644
4645     die "monitor read timeout\n" if !scalar(@ready);
4646
4647     return $res;
4648 }
4649
4650 sub qemu_block_resize {
4651     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4652
4653     my $running = check_running($vmid);
4654
4655     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4656
4657     return if !$running;
4658
4659     my $padding = (1024 - $size % 1024) % 1024;
4660     $size = $size + $padding;
4661
4662     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4663
4664 }
4665
4666 sub qemu_volume_snapshot {
4667     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4668
4669     my $running = check_running($vmid);
4670
4671     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4672         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4673     } else {
4674         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4675     }
4676 }
4677
4678 sub qemu_volume_snapshot_delete {
4679     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4680
4681     my $running = check_running($vmid);
4682
4683     if($running) {
4684
4685         $running = undef;
4686         my $conf = PVE::QemuConfig->load_config($vmid);
4687         foreach_drive($conf, sub {
4688             my ($ds, $drive) = @_;
4689             $running = 1 if $drive->{file} eq $volid;
4690         });
4691     }
4692
4693     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4694         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4695     } else {
4696         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4697     }
4698 }
4699
4700 sub set_migration_caps {
4701     my ($vmid) = @_;
4702
4703     my $cap_ref = [];
4704
4705     my $enabled_cap = {
4706         "auto-converge" => 1,
4707         "xbzrle" => 1,
4708         "x-rdma-pin-all" => 0,
4709         "zero-blocks" => 0,
4710         "compress" => 0
4711     };
4712
4713     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4714
4715     for my $supported_capability (@$supported_capabilities) {
4716         push @$cap_ref, {
4717             capability => $supported_capability->{capability},
4718             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4719         };
4720     }
4721
4722     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4723 }
4724
4725 my $fast_plug_option = {
4726     'lock' => 1,
4727     'name' => 1,
4728     'onboot' => 1,
4729     'shares' => 1,
4730     'startup' => 1,
4731     'description' => 1,
4732     'protection' => 1,
4733     'vmstatestorage' => 1,
4734     'hookscript' => 1,
4735     'tags' => 1,
4736 };
4737
4738 # hotplug changes in [PENDING]
4739 # $selection hash can be used to only apply specified options, for
4740 # example: { cores => 1 } (only apply changed 'cores')
4741 # $errors ref is used to return error messages
4742 sub vmconfig_hotplug_pending {
4743     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4744
4745     my $defaults = load_defaults();
4746     my $arch = get_vm_arch($conf);
4747     my $machine_type = get_vm_machine($conf, undef, $arch);
4748
4749     # commit values which do not have any impact on running VM first
4750     # Note: those option cannot raise errors, we we do not care about
4751     # $selection and always apply them.
4752
4753     my $add_error = sub {
4754         my ($opt, $msg) = @_;
4755         $errors->{$opt} = "hotplug problem - $msg";
4756     };
4757
4758     my $changes = 0;
4759     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4760         if ($fast_plug_option->{$opt}) {
4761             $conf->{$opt} = $conf->{pending}->{$opt};
4762             delete $conf->{pending}->{$opt};
4763             $changes = 1;
4764         }
4765     }
4766
4767     if ($changes) {
4768         PVE::QemuConfig->write_config($vmid, $conf);
4769     }
4770
4771     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4772
4773     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4774     foreach my $opt (sort keys %$pending_delete_hash) {
4775         next if $selection && !$selection->{$opt};
4776         my $force = $pending_delete_hash->{$opt}->{force};
4777         eval {
4778             if ($opt eq 'hotplug') {
4779                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4780             } elsif ($opt eq 'tablet') {
4781                 die "skip\n" if !$hotplug_features->{usb};
4782                 if ($defaults->{tablet}) {
4783                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4784                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4785                         if $arch eq 'aarch64';
4786                 } else {
4787                     vm_deviceunplug($vmid, $conf, 'tablet');
4788                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4789                 }
4790             } elsif ($opt =~ m/^usb\d+/) {
4791                 die "skip\n";
4792                 # since we cannot reliably hot unplug usb devices
4793                 # we are disabling it
4794                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4795                 vm_deviceunplug($vmid, $conf, $opt);
4796             } elsif ($opt eq 'vcpus') {
4797                 die "skip\n" if !$hotplug_features->{cpu};
4798                 qemu_cpu_hotplug($vmid, $conf, undef);
4799             } elsif ($opt eq 'balloon') {
4800                 # enable balloon device is not hotpluggable
4801                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4802                 # here we reset the ballooning value to memory
4803                 my $balloon = $conf->{memory} || $defaults->{memory};
4804                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4805             } elsif ($fast_plug_option->{$opt}) {
4806                 # do nothing
4807             } elsif ($opt =~ m/^net(\d+)$/) {
4808                 die "skip\n" if !$hotplug_features->{network};
4809                 vm_deviceunplug($vmid, $conf, $opt);
4810             } elsif (is_valid_drivename($opt)) {
4811                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4812                 vm_deviceunplug($vmid, $conf, $opt);
4813                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4814             } elsif ($opt =~ m/^memory$/) {
4815                 die "skip\n" if !$hotplug_features->{memory};
4816                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4817             } elsif ($opt eq 'cpuunits') {
4818                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4819             } elsif ($opt eq 'cpulimit') {
4820                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4821             } else {
4822                 die "skip\n";
4823             }
4824         };
4825         if (my $err = $@) {
4826             &$add_error($opt, $err) if $err ne "skip\n";
4827         } else {
4828             delete $conf->{$opt};
4829             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4830         }
4831     }
4832
4833     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4834     $apply_pending_cloudinit = sub {
4835         return if $apply_pending_cloudinit_done; # once is enough
4836         $apply_pending_cloudinit_done = 1; # once is enough
4837
4838         my ($key, $value) = @_;
4839
4840         my @cloudinit_opts = keys %$confdesc_cloudinit;
4841         foreach my $opt (keys %{$conf->{pending}}) {
4842             next if !grep { $_ eq $opt } @cloudinit_opts;
4843             $conf->{$opt} = delete $conf->{pending}->{$opt};
4844         }
4845
4846         my $new_conf = { %$conf };
4847         $new_conf->{$key} = $value;
4848         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4849     };
4850
4851     foreach my $opt (keys %{$conf->{pending}}) {
4852         next if $selection && !$selection->{$opt};
4853         my $value = $conf->{pending}->{$opt};
4854         eval {
4855             if ($opt eq 'hotplug') {
4856                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4857             } elsif ($opt eq 'tablet') {
4858                 die "skip\n" if !$hotplug_features->{usb};
4859                 if ($value == 1) {
4860                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4861                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4862                         if $arch eq 'aarch64';
4863                 } elsif ($value == 0) {
4864                     vm_deviceunplug($vmid, $conf, 'tablet');
4865                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4866                 }
4867             } elsif ($opt =~ m/^usb\d+$/) {
4868                 die "skip\n";
4869                 # since we cannot reliably hot unplug usb devices
4870                 # we are disabling it
4871                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4872                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4873                 die "skip\n" if !$d;
4874                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4875             } elsif ($opt eq 'vcpus') {
4876                 die "skip\n" if !$hotplug_features->{cpu};
4877                 qemu_cpu_hotplug($vmid, $conf, $value);
4878             } elsif ($opt eq 'balloon') {
4879                 # enable/disable balloning device is not hotpluggable
4880                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4881                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4882                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4883
4884                 # allow manual ballooning if shares is set to zero
4885                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4886                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4887                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4888                 }
4889             } elsif ($opt =~ m/^net(\d+)$/) {
4890                 # some changes can be done without hotplug
4891                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4892                                     $vmid, $opt, $value, $arch, $machine_type);
4893             } elsif (is_valid_drivename($opt)) {
4894                 die "skip\n" if $opt eq 'efidisk0';
4895                 # some changes can be done without hotplug
4896                 my $drive = parse_drive($opt, $value);
4897                 if (drive_is_cloudinit($drive)) {
4898                     &$apply_pending_cloudinit($opt, $value);
4899                 }
4900                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4901                                      $vmid, $opt, $value, $arch, $machine_type);
4902             } elsif ($opt =~ m/^memory$/) { #dimms
4903                 die "skip\n" if !$hotplug_features->{memory};
4904                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4905             } elsif ($opt eq 'cpuunits') {
4906                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4907             } elsif ($opt eq 'cpulimit') {
4908                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4909                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4910             } else {
4911                 die "skip\n";  # skip non-hot-pluggable options
4912             }
4913         };
4914         if (my $err = $@) {
4915             &$add_error($opt, $err) if $err ne "skip\n";
4916         } else {
4917             $conf->{$opt} = $value;
4918             delete $conf->{pending}->{$opt};
4919         }
4920     }
4921
4922     PVE::QemuConfig->write_config($vmid, $conf);
4923 }
4924
4925 sub try_deallocate_drive {
4926     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4927
4928     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4929         my $volid = $drive->{file};
4930         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4931             my $sid = PVE::Storage::parse_volume_id($volid);
4932             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4933
4934             # check if the disk is really unused
4935             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4936                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4937             PVE::Storage::vdisk_free($storecfg, $volid);
4938             return 1;
4939         } else {
4940             # If vm is not owner of this disk remove from config
4941             return 1;
4942         }
4943     }
4944
4945     return undef;
4946 }
4947
4948 sub vmconfig_delete_or_detach_drive {
4949     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4950
4951     my $drive = parse_drive($opt, $conf->{$opt});
4952
4953     my $rpcenv = PVE::RPCEnvironment::get();
4954     my $authuser = $rpcenv->get_user();
4955
4956     if ($force) {
4957         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4958         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4959     } else {
4960         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4961     }
4962 }
4963
4964
4965
4966 sub vmconfig_apply_pending {
4967     my ($vmid, $conf, $storecfg, $errors) = @_;
4968
4969     my $add_apply_error = sub {
4970         my ($opt, $msg) = @_;
4971         my $err_msg = "unable to apply pending change $opt : $msg";
4972         $errors->{$opt} = $err_msg;
4973         warn $err_msg;
4974     };
4975
4976     # cold plug
4977
4978     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4979     foreach my $opt (sort keys %$pending_delete_hash) {
4980         my $force = $pending_delete_hash->{$opt}->{force};
4981         eval {
4982             if ($opt =~ m/^unused/) {
4983                 die "internal error";
4984             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4985                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4986             }
4987         };
4988         if (my $err = $@) {
4989             $add_apply_error->($opt, $err);
4990         } else {
4991             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4992             delete $conf->{$opt};
4993         }
4994     }
4995
4996     PVE::QemuConfig->cleanup_pending($conf);
4997
4998     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4999         next if $opt eq 'delete'; # just to be sure
5000         eval {
5001             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5002                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5003             }
5004         };
5005         if (my $err = $@) {
5006             $add_apply_error->($opt, $err);
5007         } else {
5008             $conf->{$opt} = delete $conf->{pending}->{$opt};
5009         }
5010     }
5011
5012     # write all changes at once to avoid unnecessary i/o
5013     PVE::QemuConfig->write_config($vmid, $conf);
5014 }
5015
5016 my $safe_num_ne = sub {
5017     my ($a, $b) = @_;
5018
5019     return 0 if !defined($a) && !defined($b);
5020     return 1 if !defined($a);
5021     return 1 if !defined($b);
5022
5023     return $a != $b;
5024 };
5025
5026 my $safe_string_ne = sub {
5027     my ($a, $b) = @_;
5028
5029     return 0 if !defined($a) && !defined($b);
5030     return 1 if !defined($a);
5031     return 1 if !defined($b);
5032
5033     return $a ne $b;
5034 };
5035
5036 sub vmconfig_update_net {
5037     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5038
5039     my $newnet = parse_net($value);
5040
5041     if ($conf->{$opt}) {
5042         my $oldnet = parse_net($conf->{$opt});
5043
5044         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5045             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5046             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5047             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5048
5049             # for non online change, we try to hot-unplug
5050             die "skip\n" if !$hotplug;
5051             vm_deviceunplug($vmid, $conf, $opt);
5052         } else {
5053
5054             die "internal error" if $opt !~ m/net(\d+)/;
5055             my $iface = "tap${vmid}i$1";
5056
5057             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5058                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5059                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5060                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5061                 PVE::Network::tap_unplug($iface);
5062                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5063             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5064                 # Rate can be applied on its own but any change above needs to
5065                 # include the rate in tap_plug since OVS resets everything.
5066                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5067             }
5068
5069             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5070                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5071             }
5072
5073             return 1;
5074         }
5075     }
5076
5077     if ($hotplug) {
5078         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5079     } else {
5080         die "skip\n";
5081     }
5082 }
5083
5084 sub vmconfig_update_disk {
5085     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5086
5087     my $drive = parse_drive($opt, $value);
5088
5089     if ($conf->{$opt}) {
5090
5091         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5092
5093             my $media = $drive->{media} || 'disk';
5094             my $oldmedia = $old_drive->{media} || 'disk';
5095             die "unable to change media type\n" if $media ne $oldmedia;
5096
5097             if (!drive_is_cdrom($old_drive)) {
5098
5099                 if ($drive->{file} ne $old_drive->{file}) {
5100
5101                     die "skip\n" if !$hotplug;
5102
5103                     # unplug and register as unused
5104                     vm_deviceunplug($vmid, $conf, $opt);
5105                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5106
5107                 } else {
5108                     # update existing disk
5109
5110                     # skip non hotpluggable value
5111                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5112                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5113                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5114                         &$safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5115                         &$safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5116                         die "skip\n";
5117                     }
5118
5119                     # apply throttle
5120                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5121                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5122                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5123                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5124                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5125                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5126                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5127                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5128                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5129                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5130                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5131                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5132                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5133                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5134                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5135                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5136                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5137                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5138
5139                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5140                                                    ($drive->{mbps} || 0)*1024*1024,
5141                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5142                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5143                                                    $drive->{iops} || 0,
5144                                                    $drive->{iops_rd} || 0,
5145                                                    $drive->{iops_wr} || 0,
5146                                                    ($drive->{mbps_max} || 0)*1024*1024,
5147                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5148                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5149                                                    $drive->{iops_max} || 0,
5150                                                    $drive->{iops_rd_max} || 0,
5151                                                    $drive->{iops_wr_max} || 0,
5152                                                    $drive->{bps_max_length} || 1,
5153                                                    $drive->{bps_rd_max_length} || 1,
5154                                                    $drive->{bps_wr_max_length} || 1,
5155                                                    $drive->{iops_max_length} || 1,
5156                                                    $drive->{iops_rd_max_length} || 1,
5157                                                    $drive->{iops_wr_max_length} || 1);
5158
5159                     }
5160
5161                     return 1;
5162                 }
5163
5164             } else { # cdrom
5165
5166                 if ($drive->{file} eq 'none') {
5167                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5168                     if (drive_is_cloudinit($old_drive)) {
5169                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5170                     }
5171                 } else {
5172                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5173
5174                     # force eject if locked
5175                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5176
5177                     if ($path) {
5178                         mon_cmd($vmid, "blockdev-change-medium",
5179                             id => "$opt", filename => "$path");
5180                     }
5181                 }
5182
5183                 return 1;
5184             }
5185         }
5186     }
5187
5188     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5189     # hotplug new disks
5190     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5191     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5192 }
5193
5194 sub vm_start {
5195     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5196         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage, $timeout) = @_;
5197
5198     PVE::QemuConfig->lock_config($vmid, sub {
5199         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5200
5201         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5202
5203         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5204
5205         PVE::QemuConfig->check_lock($conf)
5206             if !($skiplock || $is_suspended);
5207
5208         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5209
5210         # clean up leftover reboot request files
5211         eval { clear_reboot_request($vmid); };
5212         warn $@ if $@;
5213
5214         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5215             vmconfig_apply_pending($vmid, $conf, $storecfg);
5216             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5217         }
5218
5219         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5220
5221         my $defaults = load_defaults();
5222
5223         # set environment variable useful inside network script
5224         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5225
5226         my $local_volumes = {};
5227
5228         if ($targetstorage) {
5229             foreach_drive($conf, sub {
5230                 my ($ds, $drive) = @_;
5231
5232                 return if drive_is_cdrom($drive);
5233
5234                 my $volid = $drive->{file};
5235
5236                 return if !$volid;
5237
5238                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5239
5240                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5241                 return if $scfg->{shared};
5242                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5243             });
5244
5245             my $format = undef;
5246
5247             foreach my $opt (sort keys %$local_volumes) {
5248
5249                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5250                 my $drive = parse_drive($opt, $conf->{$opt});
5251
5252                 # If a remote storage is specified and the format of the original
5253                 # volume is not available there, fall back to the default format.
5254                 # Otherwise use the same format as the original.
5255                 if ($targetstorage && $targetstorage ne "1") {
5256                     $storeid = $targetstorage;
5257                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5258                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5259                     my $fileFormat = qemu_img_format($scfg, $volname);
5260                     $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5261                 } else {
5262                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5263                     $format = qemu_img_format($scfg, $volname);
5264                 }
5265
5266                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5267                 my $newdrive = $drive;
5268                 $newdrive->{format} = $format;
5269                 $newdrive->{file} = $newvolid;
5270                 my $drivestr = print_drive($newdrive);
5271                 $local_volumes->{$opt} = $drivestr;
5272                 #pass drive to conf for command line
5273                 $conf->{$opt} = $drivestr;
5274             }
5275         }
5276
5277         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5278
5279         if ($is_suspended) {
5280             # enforce machine type on suspended vm to ensure HW compatibility
5281             $forcemachine = $conf->{runningmachine};
5282             print "Resuming suspended VM\n";
5283         }
5284
5285         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5286
5287         my $migration_ip;
5288         my $get_migration_ip = sub {
5289             my ($cidr, $nodename) = @_;
5290
5291             return $migration_ip if defined($migration_ip);
5292
5293             if (!defined($cidr)) {
5294                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5295                 $cidr = $dc_conf->{migration}->{network};
5296             }
5297
5298             if (defined($cidr)) {
5299                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5300
5301                 die "could not get IP: no address configured on local " .
5302                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5303
5304                 die "could not get IP: multiple addresses configured on local " .
5305                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5306
5307                 $migration_ip = @$ips[0];
5308             }
5309
5310             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5311                 if !defined($migration_ip);
5312
5313             return $migration_ip;
5314         };
5315
5316         my $migrate_uri;
5317         if ($statefile) {
5318             if ($statefile eq 'tcp') {
5319                 my $localip = "localhost";
5320                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5321                 my $nodename = nodename();
5322
5323                 if (!defined($migration_type)) {
5324                     if (defined($datacenterconf->{migration}->{type})) {
5325                         $migration_type = $datacenterconf->{migration}->{type};
5326                     } else {
5327                         $migration_type = 'secure';
5328                     }
5329                 }
5330
5331                 if ($migration_type eq 'insecure') {
5332                     $localip = $get_migration_ip->($migration_network, $nodename);
5333                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5334                 }
5335
5336                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5337                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5338                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5339                 push @$cmd, '-incoming', $migrate_uri;
5340                 push @$cmd, '-S';
5341
5342             } elsif ($statefile eq 'unix') {
5343                 # should be default for secure migrations as a ssh TCP forward
5344                 # tunnel is not deterministic reliable ready and fails regurarly
5345                 # to set up in time, so use UNIX socket forwards
5346                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5347                 unlink $socket_addr;
5348
5349                 $migrate_uri = "unix:$socket_addr";
5350
5351                 push @$cmd, '-incoming', $migrate_uri;
5352                 push @$cmd, '-S';
5353
5354             } elsif (-e $statefile) {
5355                 push @$cmd, '-loadstate', $statefile;
5356             } else {
5357                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5358                 push @$vollist, $statefile;
5359                 push @$cmd, '-loadstate', $statepath;
5360             }
5361         } elsif ($paused) {
5362             push @$cmd, '-S';
5363         }
5364
5365         # host pci devices
5366         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5367           my $d = parse_hostpci($conf->{"hostpci$i"});
5368           next if !$d;
5369           my $pcidevices = $d->{pciid};
5370           foreach my $pcidevice (@$pcidevices) {
5371                 my $pciid = $pcidevice->{id};
5372
5373                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5374                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5375                 die "no pci device info for device '$pciid'\n" if !$info;
5376
5377                 if ($d->{mdev}) {
5378                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5379                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5380                 } else {
5381                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5382                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5383                     die "can't reset pci device '$pciid'\n"
5384                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5385                 }
5386           }
5387         }
5388
5389         PVE::Storage::activate_volumes($storecfg, $vollist);
5390
5391         eval {
5392             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5393                 outfunc => sub {}, errfunc => sub {});
5394         };
5395         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5396         # timeout should be more than enough here...
5397         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5398
5399         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5400                                                   : $defaults->{cpuunits};
5401
5402         my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
5403         my %run_params = (
5404             timeout => $statefile ? undef : $start_timeout,
5405             umask => 0077,
5406             noerr => 1,
5407         );
5408
5409         # when migrating, prefix QEMU output so other side can pick up any
5410         # errors that might occur and show the user
5411         if ($migratedfrom) {
5412             $run_params{quiet} = 1;
5413             $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5414         }
5415
5416         my %properties = (
5417             Slice => 'qemu.slice',
5418             KillMode => 'none',
5419             CPUShares => $cpuunits
5420         );
5421
5422         if (my $cpulimit = $conf->{cpulimit}) {
5423             $properties{CPUQuota} = int($cpulimit * 100);
5424         }
5425         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5426
5427         my $run_qemu = sub {
5428             PVE::Tools::run_fork sub {
5429                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5430
5431                 my $exitcode = run_command($cmd, %run_params);
5432                 die "QEMU exited with code $exitcode\n" if $exitcode;
5433             };
5434         };
5435
5436         if ($conf->{hugepages}) {
5437
5438             my $code = sub {
5439                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5440                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5441
5442                 PVE::QemuServer::Memory::hugepages_mount();
5443                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5444
5445                 eval { $run_qemu->() };
5446                 if (my $err = $@) {
5447                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5448                     die $err;
5449                 }
5450
5451                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5452             };
5453             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5454
5455         } else {
5456             eval { $run_qemu->() };
5457         }
5458
5459         if (my $err = $@) {
5460             # deactivate volumes if start fails
5461             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5462             die "start failed: $err";
5463         }
5464
5465         print "migration listens on $migrate_uri\n" if $migrate_uri;
5466
5467         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5468             eval { mon_cmd($vmid, "cont"); };
5469             warn $@ if $@;
5470         }
5471
5472         #start nbd server for storage migration
5473         if ($targetstorage) {
5474             my $nodename = nodename();
5475             my $localip = $get_migration_ip->($migration_network, $nodename);
5476             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5477             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5478
5479             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5480
5481             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5482
5483             foreach my $opt (sort keys %$local_volumes) {
5484                 my $drivestr = $local_volumes->{$opt};
5485                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5486                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5487                 print "storage migration listens on $migrate_storage_uri volume:$drivestr\n";
5488             }
5489         }
5490
5491         if ($migratedfrom) {
5492             eval {
5493                 set_migration_caps($vmid);
5494             };
5495             warn $@ if $@;
5496
5497             if ($spice_port) {
5498                 print "spice listens on port $spice_port\n";
5499                 if ($spice_ticket) {
5500                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5501                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5502                 }
5503             }
5504
5505         } else {
5506             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5507                 if !$statefile && $conf->{balloon};
5508
5509             foreach my $opt (keys %$conf) {
5510                 next if $opt !~  m/^net\d+$/;
5511                 my $nicconf = parse_net($conf->{$opt});
5512                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5513             }
5514         }
5515
5516         mon_cmd($vmid, 'qom-set',
5517                     path => "machine/peripheral/balloon0",
5518                     property => "guest-stats-polling-interval",
5519                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5520
5521         if ($is_suspended) {
5522             print "Resumed VM, removing state\n";
5523             if (my $vmstate = $conf->{vmstate}) {
5524                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5525                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5526             }
5527             delete $conf->@{qw(lock vmstate runningmachine)};
5528             PVE::QemuConfig->write_config($vmid, $conf);
5529         }
5530
5531         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5532     });
5533 }
5534
5535 sub vm_commandline {
5536     my ($storecfg, $vmid, $snapname) = @_;
5537
5538     my $conf = PVE::QemuConfig->load_config($vmid);
5539     my $forcemachine;
5540
5541     if ($snapname) {
5542         my $snapshot = $conf->{snapshots}->{$snapname};
5543         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5544
5545         # check for a 'runningmachine' in snapshot
5546         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5547
5548         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5549
5550         $conf = $snapshot;
5551     }
5552
5553     my $defaults = load_defaults();
5554
5555     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5556
5557     return PVE::Tools::cmd2string($cmd);
5558 }
5559
5560 sub vm_reset {
5561     my ($vmid, $skiplock) = @_;
5562
5563     PVE::QemuConfig->lock_config($vmid, sub {
5564
5565         my $conf = PVE::QemuConfig->load_config($vmid);
5566
5567         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5568
5569         mon_cmd($vmid, "system_reset");
5570     });
5571 }
5572
5573 sub get_vm_volumes {
5574     my ($conf) = @_;
5575
5576     my $vollist = [];
5577     foreach_volid($conf, sub {
5578         my ($volid, $attr) = @_;
5579
5580         return if $volid =~ m|^/|;
5581
5582         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5583         return if !$sid;
5584
5585         push @$vollist, $volid;
5586     });
5587
5588     return $vollist;
5589 }
5590
5591 sub vm_stop_cleanup {
5592     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5593
5594     eval {
5595
5596         if (!$keepActive) {
5597             my $vollist = get_vm_volumes($conf);
5598             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5599         }
5600
5601         foreach my $ext (qw(mon qmp pid vnc qga)) {
5602             unlink "/var/run/qemu-server/${vmid}.$ext";
5603         }
5604
5605         if ($conf->{ivshmem}) {
5606             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5607             # just delete it for now, VMs which have this already open do not
5608             # are affected, but new VMs will get a separated one. If this
5609             # becomes an issue we either add some sort of ref-counting or just
5610             # add a "don't delete on stop" flag to the ivshmem format.
5611             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5612         }
5613
5614         foreach my $key (keys %$conf) {
5615             next if $key !~ m/^hostpci(\d+)$/;
5616             my $hostpciindex = $1;
5617             my $d = parse_hostpci($conf->{$key});
5618             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5619
5620             foreach my $pci (@{$d->{pciid}}) {
5621                 my $pciid = $pci->{id};
5622                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5623             }
5624         }
5625
5626         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5627     };
5628     warn $@ if $@; # avoid errors - just warn
5629 }
5630
5631 # call only in locked context
5632 sub _do_vm_stop {
5633     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5634
5635     my $pid = check_running($vmid, $nocheck);
5636     return if !$pid;
5637
5638     my $conf;
5639     if (!$nocheck) {
5640         $conf = PVE::QemuConfig->load_config($vmid);
5641         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5642         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5643             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5644             $timeout = $opts->{down} if $opts->{down};
5645         }
5646         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5647     }
5648
5649     eval {
5650         if ($shutdown) {
5651             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5652                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5653             } else {
5654                 mon_cmd($vmid, "system_powerdown");
5655             }
5656         } else {
5657             mon_cmd($vmid, "quit");
5658         }
5659     };
5660     my $err = $@;
5661
5662     if (!$err) {
5663         $timeout = 60 if !defined($timeout);
5664
5665         my $count = 0;
5666         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5667             $count++;
5668             sleep 1;
5669         }
5670
5671         if ($count >= $timeout) {
5672             if ($force) {
5673                 warn "VM still running - terminating now with SIGTERM\n";
5674                 kill 15, $pid;
5675             } else {
5676                 die "VM quit/powerdown failed - got timeout\n";
5677             }
5678         } else {
5679             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5680             return;
5681         }
5682     } else {
5683         if ($force) {
5684             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5685             kill 15, $pid;
5686         } else {
5687             die "VM quit/powerdown failed\n";
5688         }
5689     }
5690
5691     # wait again
5692     $timeout = 10;
5693
5694     my $count = 0;
5695     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5696         $count++;
5697         sleep 1;
5698     }
5699
5700     if ($count >= $timeout) {
5701         warn "VM still running - terminating now with SIGKILL\n";
5702         kill 9, $pid;
5703         sleep 1;
5704     }
5705
5706     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5707 }
5708
5709 # Note: use $nocheck to skip tests if VM configuration file exists.
5710 # We need that when migration VMs to other nodes (files already moved)
5711 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5712 sub vm_stop {
5713     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5714
5715     $force = 1 if !defined($force) && !$shutdown;
5716
5717     if ($migratedfrom){
5718         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5719         kill 15, $pid if $pid;
5720         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5721         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5722         return;
5723     }
5724
5725     PVE::QemuConfig->lock_config($vmid, sub {
5726         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5727    });
5728 }
5729
5730 sub vm_reboot {
5731     my ($vmid, $timeout) = @_;
5732
5733     PVE::QemuConfig->lock_config($vmid, sub {
5734         eval {
5735
5736             # only reboot if running, as qmeventd starts it again on a stop event
5737             return if !check_running($vmid);
5738
5739             create_reboot_request($vmid);
5740
5741             my $storecfg = PVE::Storage::config();
5742             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5743
5744         };
5745         if (my $err = $@) {
5746             # avoid that the next normal shutdown will be confused for a reboot
5747             clear_reboot_request($vmid);
5748             die $err;
5749         }
5750    });
5751 }
5752
5753 # note: if using the statestorage parameter, the caller has to check privileges
5754 sub vm_suspend {
5755     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5756
5757     my $conf;
5758     my $path;
5759     my $storecfg;
5760     my $vmstate;
5761
5762     PVE::QemuConfig->lock_config($vmid, sub {
5763
5764         $conf = PVE::QemuConfig->load_config($vmid);
5765
5766         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5767         PVE::QemuConfig->check_lock($conf)
5768             if !($skiplock || $is_backing_up);
5769
5770         die "cannot suspend to disk during backup\n"
5771             if $is_backing_up && $includestate;
5772
5773         if ($includestate) {
5774             $conf->{lock} = 'suspending';
5775             my $date = strftime("%Y-%m-%d", localtime(time()));
5776             $storecfg = PVE::Storage::config();
5777             if (!$statestorage) {
5778                 $statestorage = find_vmstate_storage($conf, $storecfg);
5779                 # check permissions for the storage
5780                 my $rpcenv = PVE::RPCEnvironment::get();
5781                 if ($rpcenv->{type} ne 'cli') {
5782                     my $authuser = $rpcenv->get_user();
5783                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5784                 }
5785             }
5786
5787
5788             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5789             $path = PVE::Storage::path($storecfg, $vmstate);
5790             PVE::QemuConfig->write_config($vmid, $conf);
5791         } else {
5792             mon_cmd($vmid, "stop");
5793         }
5794     });
5795
5796     if ($includestate) {
5797         # save vm state
5798         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5799
5800         eval {
5801             mon_cmd($vmid, "savevm-start", statefile => $path);
5802             for(;;) {
5803                 my $state = mon_cmd($vmid, "query-savevm");
5804                 if (!$state->{status}) {
5805                     die "savevm not active\n";
5806                 } elsif ($state->{status} eq 'active') {
5807                     sleep(1);
5808                     next;
5809                 } elsif ($state->{status} eq 'completed') {
5810                     print "State saved, quitting\n";
5811                     last;
5812                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5813                     die "query-savevm failed with error '$state->{error}'\n"
5814                 } else {
5815                     die "query-savevm returned status '$state->{status}'\n";
5816                 }
5817             }
5818         };
5819         my $err = $@;
5820
5821         PVE::QemuConfig->lock_config($vmid, sub {
5822             $conf = PVE::QemuConfig->load_config($vmid);
5823             if ($err) {
5824                 # cleanup, but leave suspending lock, to indicate something went wrong
5825                 eval {
5826                     mon_cmd($vmid, "savevm-end");
5827                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5828                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5829                     delete $conf->@{qw(vmstate runningmachine)};
5830                     PVE::QemuConfig->write_config($vmid, $conf);
5831                 };
5832                 warn $@ if $@;
5833                 die $err;
5834             }
5835
5836             die "lock changed unexpectedly\n"
5837                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5838
5839             mon_cmd($vmid, "quit");
5840             $conf->{lock} = 'suspended';
5841             PVE::QemuConfig->write_config($vmid, $conf);
5842         });
5843     }
5844 }
5845
5846 sub vm_resume {
5847     my ($vmid, $skiplock, $nocheck) = @_;
5848
5849     PVE::QemuConfig->lock_config($vmid, sub {
5850         my $res = mon_cmd($vmid, 'query-status');
5851         my $resume_cmd = 'cont';
5852
5853         if ($res->{status} && $res->{status} eq 'suspended') {
5854             $resume_cmd = 'system_wakeup';
5855         }
5856
5857         if (!$nocheck) {
5858
5859             my $conf = PVE::QemuConfig->load_config($vmid);
5860
5861             PVE::QemuConfig->check_lock($conf)
5862                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5863         }
5864
5865         mon_cmd($vmid, $resume_cmd);
5866     });
5867 }
5868
5869 sub vm_sendkey {
5870     my ($vmid, $skiplock, $key) = @_;
5871
5872     PVE::QemuConfig->lock_config($vmid, sub {
5873
5874         my $conf = PVE::QemuConfig->load_config($vmid);
5875
5876         # there is no qmp command, so we use the human monitor command
5877         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5878         die $res if $res ne '';
5879     });
5880 }
5881
5882 # vzdump restore implementaion
5883
5884 sub tar_archive_read_firstfile {
5885     my $archive = shift;
5886
5887     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5888
5889     # try to detect archive type first
5890     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5891         die "unable to open file '$archive'\n";
5892     my $firstfile = <$fh>;
5893     kill 15, $pid;
5894     close $fh;
5895
5896     die "ERROR: archive contaions no data\n" if !$firstfile;
5897     chomp $firstfile;
5898
5899     return $firstfile;
5900 }
5901
5902 sub tar_restore_cleanup {
5903     my ($storecfg, $statfile) = @_;
5904
5905     print STDERR "starting cleanup\n";
5906
5907     if (my $fd = IO::File->new($statfile, "r")) {
5908         while (defined(my $line = <$fd>)) {
5909             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5910                 my $volid = $2;
5911                 eval {
5912                     if ($volid =~ m|^/|) {
5913                         unlink $volid || die 'unlink failed\n';
5914                     } else {
5915                         PVE::Storage::vdisk_free($storecfg, $volid);
5916                     }
5917                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5918                 };
5919                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5920             } else {
5921                 print STDERR "unable to parse line in statfile - $line";
5922             }
5923         }
5924         $fd->close();
5925     }
5926 }
5927
5928 sub restore_archive {
5929     my ($archive, $vmid, $user, $opts) = @_;
5930
5931     my $format = $opts->{format};
5932     my $comp;
5933
5934     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5935         $format = 'tar' if !$format;
5936         $comp = 'gzip';
5937     } elsif ($archive =~ m/\.tar$/) {
5938         $format = 'tar' if !$format;
5939     } elsif ($archive =~ m/.tar.lzo$/) {
5940         $format = 'tar' if !$format;
5941         $comp = 'lzop';
5942     } elsif ($archive =~ m/\.vma$/) {
5943         $format = 'vma' if !$format;
5944     } elsif ($archive =~ m/\.vma\.gz$/) {
5945         $format = 'vma' if !$format;
5946         $comp = 'gzip';
5947     } elsif ($archive =~ m/\.vma\.lzo$/) {
5948         $format = 'vma' if !$format;
5949         $comp = 'lzop';
5950     } else {
5951         $format = 'vma' if !$format; # default
5952     }
5953
5954     # try to detect archive format
5955     if ($format eq 'tar') {
5956         return restore_tar_archive($archive, $vmid, $user, $opts);
5957     } else {
5958         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5959     }
5960 }
5961
5962 sub restore_update_config_line {
5963     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5964
5965     return if $line =~ m/^\#qmdump\#/;
5966     return if $line =~ m/^\#vzdump\#/;
5967     return if $line =~ m/^lock:/;
5968     return if $line =~ m/^unused\d+:/;
5969     return if $line =~ m/^parent:/;
5970
5971     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5972     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5973         # try to convert old 1.X settings
5974         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5975         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5976             my ($model, $macaddr) = split(/\=/, $devconfig);
5977             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5978             my $net = {
5979                 model => $model,
5980                 bridge => "vmbr$ind",
5981                 macaddr => $macaddr,
5982             };
5983             my $netstr = print_net($net);
5984
5985             print $outfd "net$cookie->{netcount}: $netstr\n";
5986             $cookie->{netcount}++;
5987         }
5988     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5989         my ($id, $netstr) = ($1, $2);
5990         my $net = parse_net($netstr);
5991         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5992         $netstr = print_net($net);
5993         print $outfd "$id: $netstr\n";
5994     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5995         my $virtdev = $1;
5996         my $value = $3;
5997         my $di = parse_drive($virtdev, $value);
5998         if (defined($di->{backup}) && !$di->{backup}) {
5999             print $outfd "#$line";
6000         } elsif ($map->{$virtdev}) {
6001             delete $di->{format}; # format can change on restore
6002             $di->{file} = $map->{$virtdev};
6003             $value = print_drive($di);
6004             print $outfd "$virtdev: $value\n";
6005         } else {
6006             print $outfd $line;
6007         }
6008     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6009         my $vmgenid = $1;
6010         if ($vmgenid ne '0') {
6011             # always generate a new vmgenid if there was a valid one setup
6012             $vmgenid = generate_uuid();
6013         }
6014         print $outfd "vmgenid: $vmgenid\n";
6015     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6016         my ($uuid, $uuid_str);
6017         UUID::generate($uuid);
6018         UUID::unparse($uuid, $uuid_str);
6019         my $smbios1 = parse_smbios1($2);
6020         $smbios1->{uuid} = $uuid_str;
6021         print $outfd $1.print_smbios1($smbios1)."\n";
6022     } else {
6023         print $outfd $line;
6024     }
6025 }
6026
6027 sub scan_volids {
6028     my ($cfg, $vmid) = @_;
6029
6030     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6031
6032     my $volid_hash = {};
6033     foreach my $storeid (keys %$info) {
6034         foreach my $item (@{$info->{$storeid}}) {
6035             next if !($item->{volid} && $item->{size});
6036             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6037             $volid_hash->{$item->{volid}} = $item;
6038         }
6039     }
6040
6041     return $volid_hash;
6042 }
6043
6044 sub is_volume_in_use {
6045     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6046
6047     my $path = PVE::Storage::path($storecfg, $volid);
6048
6049     my $scan_config = sub {
6050         my ($cref, $snapname) = @_;
6051
6052         foreach my $key (keys %$cref) {
6053             my $value = $cref->{$key};
6054             if (is_valid_drivename($key)) {
6055                 next if $skip_drive && $key eq $skip_drive;
6056                 my $drive = parse_drive($key, $value);
6057                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6058                 return 1 if $volid eq $drive->{file};
6059                 if ($drive->{file} =~ m!^/!) {
6060                     return 1 if $drive->{file} eq $path;
6061                 } else {
6062                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6063                     next if !$storeid;
6064                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6065                     next if !$scfg;
6066                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6067                 }
6068             }
6069         }
6070
6071         return 0;
6072     };
6073
6074     return 1 if &$scan_config($conf);
6075
6076     undef $skip_drive;
6077
6078     foreach my $snapname (keys %{$conf->{snapshots}}) {
6079         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6080     }
6081
6082     return 0;
6083 }
6084
6085 sub update_disksize {
6086     my ($drive, $volid_hash) = @_;
6087
6088     my $volid = $drive->{file};
6089     return undef if !defined($volid);
6090
6091     my $oldsize = $drive->{size};
6092     my $newsize = $volid_hash->{$volid}->{size};
6093
6094     if (defined($newsize) && defined($oldsize) && $newsize != $oldsize) {
6095         $drive->{size} = $newsize;
6096
6097         my $old_fmt = PVE::JSONSchema::format_size($oldsize);
6098         my $new_fmt = PVE::JSONSchema::format_size($newsize);
6099
6100         return wantarray ? ($drive, $old_fmt, $new_fmt) : $drive;
6101     }
6102
6103     return undef;
6104 }
6105
6106 sub update_disk_config {
6107     my ($vmid, $conf, $volid_hash) = @_;
6108
6109     my $changes;
6110     my $prefix = "VM $vmid:";
6111
6112     # used and unused disks
6113     my $referenced = {};
6114
6115     # Note: it is allowed to define multiple storages with same path (alias), so
6116     # we need to check both 'volid' and real 'path' (two different volid can point
6117     # to the same path).
6118
6119     my $referencedpath = {};
6120
6121     # update size info
6122     foreach my $opt (keys %$conf) {
6123         if (is_valid_drivename($opt)) {
6124             my $drive = parse_drive($opt, $conf->{$opt});
6125             my $volid = $drive->{file};
6126             next if !$volid;
6127
6128             # mark volid as "in-use" for next step
6129             $referenced->{$volid} = 1;
6130             if ($volid_hash->{$volid} &&
6131                 (my $path = $volid_hash->{$volid}->{path})) {
6132                 $referencedpath->{$path} = 1;
6133             }
6134
6135             next if drive_is_cdrom($drive);
6136             next if !$volid_hash->{$volid};
6137
6138             my ($updated, $old_size, $new_size) = update_disksize($drive, $volid_hash);
6139             if (defined($updated)) {
6140                 $changes = 1;
6141                 $conf->{$opt} = print_drive($updated);
6142                 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
6143             }
6144         }
6145     }
6146
6147     # remove 'unusedX' entry if volume is used
6148     foreach my $opt (keys %$conf) {
6149         next if $opt !~ m/^unused\d+$/;
6150         my $volid = $conf->{$opt};
6151         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6152         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6153             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6154             $changes = 1;
6155             delete $conf->{$opt};
6156         }
6157
6158         $referenced->{$volid} = 1;
6159         $referencedpath->{$path} = 1 if $path;
6160     }
6161
6162     foreach my $volid (sort keys %$volid_hash) {
6163         next if $volid =~ m/vm-$vmid-state-/;
6164         next if $referenced->{$volid};
6165         my $path = $volid_hash->{$volid}->{path};
6166         next if !$path; # just to be sure
6167         next if $referencedpath->{$path};
6168         $changes = 1;
6169         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6170         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6171         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6172     }
6173
6174     return $changes;
6175 }
6176
6177 sub rescan {
6178     my ($vmid, $nolock, $dryrun) = @_;
6179
6180     my $cfg = PVE::Storage::config();
6181
6182     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6183     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6184     foreach my $stor (keys %{$cfg->{ids}}) {
6185         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6186     }
6187
6188     print "rescan volumes...\n";
6189     my $volid_hash = scan_volids($cfg, $vmid);
6190
6191     my $updatefn =  sub {
6192         my ($vmid) = @_;
6193
6194         my $conf = PVE::QemuConfig->load_config($vmid);
6195
6196         PVE::QemuConfig->check_lock($conf);
6197
6198         my $vm_volids = {};
6199         foreach my $volid (keys %$volid_hash) {
6200             my $info = $volid_hash->{$volid};
6201             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6202         }
6203
6204         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6205
6206         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6207     };
6208
6209     if (defined($vmid)) {
6210         if ($nolock) {
6211             &$updatefn($vmid);
6212         } else {
6213             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6214         }
6215     } else {
6216         my $vmlist = config_list();
6217         foreach my $vmid (keys %$vmlist) {
6218             if ($nolock) {
6219                 &$updatefn($vmid);
6220             } else {
6221                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6222             }
6223         }
6224     }
6225 }
6226
6227 sub restore_vma_archive {
6228     my ($archive, $vmid, $user, $opts, $comp) = @_;
6229
6230     my $readfrom = $archive;
6231
6232     my $cfg = PVE::Storage::config();
6233     my $commands = [];
6234     my $bwlimit = $opts->{bwlimit};
6235
6236     my $dbg_cmdstring = '';
6237     my $add_pipe = sub {
6238         my ($cmd) = @_;
6239         push @$commands, $cmd;
6240         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6241         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6242         $readfrom = '-';
6243     };
6244
6245     my $input = undef;
6246     if ($archive eq '-') {
6247         $input = '<&STDIN';
6248     } else {
6249         # If we use a backup from a PVE defined storage we also consider that
6250         # storage's rate limit:
6251         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6252         if (defined($volid)) {
6253             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6254             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6255             if ($readlimit) {
6256                 print STDERR "applying read rate limit: $readlimit\n";
6257                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6258                 $add_pipe->($cstream);
6259             }
6260         }
6261     }
6262
6263     if ($comp) {
6264         my $cmd;
6265         if ($comp eq 'gzip') {
6266             $cmd = ['zcat', $readfrom];
6267         } elsif ($comp eq 'lzop') {
6268             $cmd = ['lzop', '-d', '-c', $readfrom];
6269         } else {
6270             die "unknown compression method '$comp'\n";
6271         }
6272         $add_pipe->($cmd);
6273     }
6274
6275     my $tmpdir = "/var/tmp/vzdumptmp$$";
6276     rmtree $tmpdir;
6277
6278     # disable interrupts (always do cleanups)
6279     local $SIG{INT} =
6280         local $SIG{TERM} =
6281         local $SIG{QUIT} =
6282         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6283
6284     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6285     POSIX::mkfifo($mapfifo, 0600);
6286     my $fifofh;
6287
6288     my $openfifo = sub {
6289         open($fifofh, '>', $mapfifo) || die $!;
6290     };
6291
6292     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6293
6294     my $oldtimeout;
6295     my $timeout = 5;
6296
6297     my $devinfo = {};
6298
6299     my $rpcenv = PVE::RPCEnvironment::get();
6300
6301     my $conffile = PVE::QemuConfig->config_file($vmid);
6302     my $tmpfn = "$conffile.$$.tmp";
6303
6304     # Note: $oldconf is undef if VM does not exist
6305     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6306     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6307
6308     my %storage_limits;
6309
6310     my $print_devmap = sub {
6311         my $virtdev_hash = {};
6312
6313         my $cfgfn = "$tmpdir/qemu-server.conf";
6314
6315         # we can read the config - that is already extracted
6316         my $fh = IO::File->new($cfgfn, "r") ||
6317             "unable to read qemu-server.conf - $!\n";
6318
6319         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6320         if (-f $fwcfgfn) {
6321             my $pve_firewall_dir = '/etc/pve/firewall';
6322             mkdir $pve_firewall_dir; # make sure the dir exists
6323             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6324         }
6325
6326         while (defined(my $line = <$fh>)) {
6327             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6328                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6329                 die "archive does not contain data for drive '$virtdev'\n"
6330                     if !$devinfo->{$devname};
6331                 if (defined($opts->{storage})) {
6332                     $storeid = $opts->{storage} || 'local';
6333                 } elsif (!$storeid) {
6334                     $storeid = 'local';
6335                 }
6336                 $format = 'raw' if !$format;
6337                 $devinfo->{$devname}->{devname} = $devname;
6338                 $devinfo->{$devname}->{virtdev} = $virtdev;
6339                 $devinfo->{$devname}->{format} = $format;
6340                 $devinfo->{$devname}->{storeid} = $storeid;
6341
6342                 # check permission on storage
6343                 my $pool = $opts->{pool}; # todo: do we need that?
6344                 if ($user ne 'root@pam') {
6345                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6346                 }
6347
6348                 $storage_limits{$storeid} = $bwlimit;
6349
6350                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6351             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6352                 my $virtdev = $1;
6353                 my $drive = parse_drive($virtdev, $2);
6354                 if (drive_is_cloudinit($drive)) {
6355                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6356                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6357                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6358
6359                     my $d = {
6360                         format => $format,
6361                         storeid => $opts->{storage} // $storeid,
6362                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6363                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6364                         name => "vm-$vmid-cloudinit",
6365                         is_cloudinit => 1,
6366                     };
6367                     $virtdev_hash->{$virtdev} = $d;
6368                 }
6369             }
6370         }
6371
6372         foreach my $key (keys %storage_limits) {
6373             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6374             next if !$limit;
6375             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6376             $storage_limits{$key} = $limit * 1024;
6377         }
6378
6379         foreach my $devname (keys %$devinfo) {
6380             die "found no device mapping information for device '$devname'\n"
6381                 if !$devinfo->{$devname}->{virtdev};
6382         }
6383
6384         # create empty/temp config
6385         if ($oldconf) {
6386             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6387             foreach_drive($oldconf, sub {
6388                 my ($ds, $drive) = @_;
6389
6390                 return if drive_is_cdrom($drive, 1);
6391
6392                 my $volid = $drive->{file};
6393                 return if !$volid || $volid =~ m|^/|;
6394
6395                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6396                 return if !$path || !$owner || ($owner != $vmid);
6397
6398                 # Note: only delete disk we want to restore
6399                 # other volumes will become unused
6400                 if ($virtdev_hash->{$ds}) {
6401                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6402                     if (my $err = $@) {
6403                         warn $err;
6404                     }
6405                 }
6406             });
6407
6408             # delete vmstate files, after the restore we have no snapshots anymore
6409             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6410                 my $snap = $oldconf->{snapshots}->{$snapname};
6411                 if ($snap->{vmstate}) {
6412                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6413                     if (my $err = $@) {
6414                         warn $err;
6415                     }
6416                 }
6417             }
6418         }
6419
6420         my $map = {};
6421         foreach my $virtdev (sort keys %$virtdev_hash) {
6422             my $d = $virtdev_hash->{$virtdev};
6423             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6424             my $storeid = $d->{storeid};
6425             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6426
6427             my $map_opts = '';
6428             if (my $limit = $storage_limits{$storeid}) {
6429                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6430             }
6431
6432             # test if requested format is supported
6433             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6434             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6435             $d->{format} = $defFormat if !$supported;
6436
6437             my $name;
6438             if ($d->{is_cloudinit}) {
6439                 $name = $d->{name};
6440                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6441             }
6442
6443             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6444             print STDERR "new volume ID is '$volid'\n";
6445             $d->{volid} = $volid;
6446
6447             PVE::Storage::activate_volumes($cfg, [$volid]);
6448
6449             my $write_zeros = 1;
6450             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6451                 $write_zeros = 0;
6452             }
6453
6454             if (!$d->{is_cloudinit}) {
6455                 my $path = PVE::Storage::path($cfg, $volid);
6456
6457                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6458
6459                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6460             }
6461             $map->{$virtdev} = $volid;
6462         }
6463
6464         $fh->seek(0, 0) || die "seek failed - $!\n";
6465
6466         my $outfd = new IO::File ($tmpfn, "w") ||
6467             die "unable to write config for VM $vmid\n";
6468
6469         my $cookie = { netcount => 0 };
6470         while (defined(my $line = <$fh>)) {
6471             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6472         }
6473
6474         $fh->close();
6475         $outfd->close();
6476     };
6477
6478     eval {
6479         # enable interrupts
6480         local $SIG{INT} =
6481             local $SIG{TERM} =
6482             local $SIG{QUIT} =
6483             local $SIG{HUP} =
6484             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6485         local $SIG{ALRM} = sub { die "got timeout\n"; };
6486
6487         $oldtimeout = alarm($timeout);
6488
6489         my $parser = sub {
6490             my $line = shift;
6491
6492             print "$line\n";
6493
6494             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6495                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6496                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6497             } elsif ($line =~ m/^CTIME: /) {
6498                 # we correctly received the vma config, so we can disable
6499                 # the timeout now for disk allocation (set to 10 minutes, so
6500                 # that we always timeout if something goes wrong)
6501                 alarm(600);
6502                 &$print_devmap();
6503                 print $fifofh "done\n";
6504                 my $tmp = $oldtimeout || 0;
6505                 $oldtimeout = undef;
6506                 alarm($tmp);
6507                 close($fifofh);
6508             }
6509         };
6510
6511         print "restore vma archive: $dbg_cmdstring\n";
6512         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6513     };
6514     my $err = $@;
6515
6516     alarm($oldtimeout) if $oldtimeout;
6517
6518     my $vollist = [];
6519     foreach my $devname (keys %$devinfo) {
6520         my $volid = $devinfo->{$devname}->{volid};
6521         push @$vollist, $volid if $volid;
6522     }
6523
6524     PVE::Storage::deactivate_volumes($cfg, $vollist);
6525
6526     unlink $mapfifo;
6527
6528     if ($err) {
6529         rmtree $tmpdir;
6530         unlink $tmpfn;
6531
6532         foreach my $devname (keys %$devinfo) {
6533             my $volid = $devinfo->{$devname}->{volid};
6534             next if !$volid;
6535             eval {
6536                 if ($volid =~ m|^/|) {
6537                     unlink $volid || die 'unlink failed\n';
6538                 } else {
6539                     PVE::Storage::vdisk_free($cfg, $volid);
6540                 }
6541                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6542             };
6543             print STDERR "unable to cleanup '$volid' - $@" if $@;
6544         }
6545         die $err;
6546     }
6547
6548     rmtree $tmpdir;
6549
6550     rename($tmpfn, $conffile) ||
6551         die "unable to commit configuration file '$conffile'\n";
6552
6553     PVE::Cluster::cfs_update(); # make sure we read new file
6554
6555     eval { rescan($vmid, 1); };
6556     warn $@ if $@;
6557 }
6558
6559 sub restore_tar_archive {
6560     my ($archive, $vmid, $user, $opts) = @_;
6561
6562     if ($archive ne '-') {
6563         my $firstfile = tar_archive_read_firstfile($archive);
6564         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6565             if $firstfile ne 'qemu-server.conf';
6566     }
6567
6568     my $storecfg = PVE::Storage::config();
6569
6570     # avoid zombie disks when restoring over an existing VM -> cleanup first
6571     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6572     # skiplock=1 because qmrestore has set the 'create' lock itself already
6573     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6574     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6575
6576     my $tocmd = "/usr/lib/qemu-server/qmextract";
6577
6578     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6579     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6580     $tocmd .= ' --prealloc' if $opts->{prealloc};
6581     $tocmd .= ' --info' if $opts->{info};
6582
6583     # tar option "xf" does not autodetect compression when read from STDIN,
6584     # so we pipe to zcat
6585     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6586         PVE::Tools::shellquote("--to-command=$tocmd");
6587
6588     my $tmpdir = "/var/tmp/vzdumptmp$$";
6589     mkpath $tmpdir;
6590
6591     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6592     local $ENV{VZDUMP_VMID} = $vmid;
6593     local $ENV{VZDUMP_USER} = $user;
6594
6595     my $conffile = PVE::QemuConfig->config_file($vmid);
6596     my $tmpfn = "$conffile.$$.tmp";
6597
6598     # disable interrupts (always do cleanups)
6599     local $SIG{INT} =
6600         local $SIG{TERM} =
6601         local $SIG{QUIT} =
6602         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6603
6604     eval {
6605         # enable interrupts
6606         local $SIG{INT} =
6607             local $SIG{TERM} =
6608             local $SIG{QUIT} =
6609             local $SIG{HUP} =
6610             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6611
6612         if ($archive eq '-') {
6613             print "extracting archive from STDIN\n";
6614             run_command($cmd, input => "<&STDIN");
6615         } else {
6616             print "extracting archive '$archive'\n";
6617             run_command($cmd);
6618         }
6619
6620         return if $opts->{info};
6621
6622         # read new mapping
6623         my $map = {};
6624         my $statfile = "$tmpdir/qmrestore.stat";
6625         if (my $fd = IO::File->new($statfile, "r")) {
6626             while (defined (my $line = <$fd>)) {
6627                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6628                     $map->{$1} = $2 if $1;
6629                 } else {
6630                     print STDERR "unable to parse line in statfile - $line\n";
6631                 }
6632             }
6633             $fd->close();
6634         }
6635
6636         my $confsrc = "$tmpdir/qemu-server.conf";
6637
6638         my $srcfd = new IO::File($confsrc, "r") ||
6639             die "unable to open file '$confsrc'\n";
6640
6641         my $outfd = new IO::File ($tmpfn, "w") ||
6642             die "unable to write config for VM $vmid\n";
6643
6644         my $cookie = { netcount => 0 };
6645         while (defined (my $line = <$srcfd>)) {
6646             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6647         }
6648
6649         $srcfd->close();
6650         $outfd->close();
6651     };
6652     if (my $err = $@) {
6653         unlink $tmpfn;
6654         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6655         die $err;
6656     }
6657
6658     rmtree $tmpdir;
6659
6660     rename $tmpfn, $conffile ||
6661         die "unable to commit configuration file '$conffile'\n";
6662
6663     PVE::Cluster::cfs_update(); # make sure we read new file
6664
6665     eval { rescan($vmid, 1); };
6666     warn $@ if $@;
6667 };
6668
6669 sub foreach_storage_used_by_vm {
6670     my ($conf, $func) = @_;
6671
6672     my $sidhash = {};
6673
6674     foreach_drive($conf, sub {
6675         my ($ds, $drive) = @_;
6676         return if drive_is_cdrom($drive);
6677
6678         my $volid = $drive->{file};
6679
6680         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6681         $sidhash->{$sid} = $sid if $sid;
6682     });
6683
6684     foreach my $sid (sort keys %$sidhash) {
6685         &$func($sid);
6686     }
6687 }
6688
6689 my $qemu_snap_storage = {
6690     rbd => 1,
6691 };
6692 sub do_snapshots_with_qemu {
6693     my ($storecfg, $volid) = @_;
6694
6695     my $storage_name = PVE::Storage::parse_volume_id($volid);
6696     my $scfg = $storecfg->{ids}->{$storage_name};
6697
6698     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6699         return 1;
6700     }
6701
6702     if ($volid =~ m/\.(qcow2|qed)$/){
6703         return 1;
6704     }
6705
6706     return undef;
6707 }
6708
6709 sub qga_check_running {
6710     my ($vmid, $nowarn) = @_;
6711
6712     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6713     if ($@) {
6714         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6715         return 0;
6716     }
6717     return 1;
6718 }
6719
6720 sub template_create {
6721     my ($vmid, $conf, $disk) = @_;
6722
6723     my $storecfg = PVE::Storage::config();
6724
6725     foreach_drive($conf, sub {
6726         my ($ds, $drive) = @_;
6727
6728         return if drive_is_cdrom($drive);
6729         return if $disk && $ds ne $disk;
6730
6731         my $volid = $drive->{file};
6732         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6733
6734         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6735         $drive->{file} = $voliddst;
6736         $conf->{$ds} = print_drive($drive);
6737         PVE::QemuConfig->write_config($vmid, $conf);
6738     });
6739 }
6740
6741 sub convert_iscsi_path {
6742     my ($path) = @_;
6743
6744     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6745         my $portal = $1;
6746         my $target = $2;
6747         my $lun = $3;
6748
6749         my $initiator_name = get_initiator_name();
6750
6751         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6752                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6753     }
6754
6755     die "cannot convert iscsi path '$path', unkown format\n";
6756 }
6757
6758 sub qemu_img_convert {
6759     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6760
6761     my $storecfg = PVE::Storage::config();
6762     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6763     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6764
6765     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6766
6767     my $cachemode;
6768     my $src_path;
6769     my $src_is_iscsi = 0;
6770     my $src_format;
6771
6772     if ($src_storeid) {
6773         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6774         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6775         $src_format = qemu_img_format($src_scfg, $src_volname);
6776         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6777         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6778         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6779     } elsif (-f $src_volid) {
6780         $src_path = $src_volid;
6781         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6782             $src_format = $1;
6783         }
6784     }
6785
6786     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6787
6788     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6789     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6790     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6791     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6792
6793     my $cmd = [];
6794     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6795     push @$cmd, '-l', "snapshot.name=$snapname"
6796         if $snapname && $src_format && $src_format eq "qcow2";
6797     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6798     push @$cmd, '-T', $cachemode if defined($cachemode);
6799
6800     if ($src_is_iscsi) {
6801         push @$cmd, '--image-opts';
6802         $src_path = convert_iscsi_path($src_path);
6803     } elsif ($src_format) {
6804         push @$cmd, '-f', $src_format;
6805     }
6806
6807     if ($dst_is_iscsi) {
6808         push @$cmd, '--target-image-opts';
6809         $dst_path = convert_iscsi_path($dst_path);
6810     } else {
6811         push @$cmd, '-O', $dst_format;
6812     }
6813
6814     push @$cmd, $src_path;
6815
6816     if (!$dst_is_iscsi && $is_zero_initialized) {
6817         push @$cmd, "zeroinit:$dst_path";
6818     } else {
6819         push @$cmd, $dst_path;
6820     }
6821
6822     my $parser = sub {
6823         my $line = shift;
6824         if($line =~ m/\((\S+)\/100\%\)/){
6825             my $percent = $1;
6826             my $transferred = int($size * $percent / 100);
6827             my $remaining = $size - $transferred;
6828
6829             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6830         }
6831
6832     };
6833
6834     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6835     my $err = $@;
6836     die "copy failed: $err" if $err;
6837 }
6838
6839 sub qemu_img_format {
6840     my ($scfg, $volname) = @_;
6841
6842     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6843         return $1;
6844     } else {
6845         return "raw";
6846     }
6847 }
6848
6849 sub qemu_drive_mirror {
6850     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6851
6852     $jobs = {} if !$jobs;
6853
6854     my $qemu_target;
6855     my $format;
6856     $jobs->{"drive-$drive"} = {};
6857
6858     if ($dst_volid =~ /^nbd:/) {
6859         $qemu_target = $dst_volid;
6860         $format = "nbd";
6861     } else {
6862         my $storecfg = PVE::Storage::config();
6863         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6864
6865         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6866
6867         $format = qemu_img_format($dst_scfg, $dst_volname);
6868
6869         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6870
6871         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6872     }
6873
6874     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6875     $opts->{format} = $format if $format;
6876
6877     if (defined($bwlimit)) {
6878         $opts->{speed} = $bwlimit * 1024;
6879         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6880     } else {
6881         print "drive mirror is starting for drive-$drive\n";
6882     }
6883
6884     # if a job already runs for this device we get an error, catch it for cleanup
6885     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6886     if (my $err = $@) {
6887         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6888         warn "$@\n" if $@;
6889         die "mirroring error: $err\n";
6890     }
6891
6892     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6893 }
6894
6895 sub qemu_drive_mirror_monitor {
6896     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6897
6898     eval {
6899         my $err_complete = 0;
6900
6901         while (1) {
6902             die "storage migration timed out\n" if $err_complete > 300;
6903
6904             my $stats = mon_cmd($vmid, "query-block-jobs");
6905
6906             my $running_mirror_jobs = {};
6907             foreach my $stat (@$stats) {
6908                 next if $stat->{type} ne 'mirror';
6909                 $running_mirror_jobs->{$stat->{device}} = $stat;
6910             }
6911
6912             my $readycounter = 0;
6913
6914             foreach my $job (keys %$jobs) {
6915
6916                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6917                     print "$job : finished\n";
6918                     delete $jobs->{$job};
6919                     next;
6920                 }
6921
6922                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6923
6924                 my $busy = $running_mirror_jobs->{$job}->{busy};
6925                 my $ready = $running_mirror_jobs->{$job}->{ready};
6926                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6927                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6928                     my $remaining = $total - $transferred;
6929                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6930
6931                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6932                 }
6933
6934                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6935             }
6936
6937             last if scalar(keys %$jobs) == 0;
6938
6939             if ($readycounter == scalar(keys %$jobs)) {
6940                 print "all mirroring jobs are ready \n";
6941                 last if $skipcomplete; #do the complete later
6942
6943                 if ($vmiddst && $vmiddst != $vmid) {
6944                     my $agent_running = $qga && qga_check_running($vmid);
6945                     if ($agent_running) {
6946                         print "freeze filesystem\n";
6947                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6948                     } else {
6949                         print "suspend vm\n";
6950                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6951                     }
6952
6953                     # if we clone a disk for a new target vm, we don't switch the disk
6954                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6955
6956                     if ($agent_running) {
6957                         print "unfreeze filesystem\n";
6958                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6959                     } else {
6960                         print "resume vm\n";
6961                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6962                     }
6963
6964                     last;
6965                 } else {
6966
6967                     foreach my $job (keys %$jobs) {
6968                         # try to switch the disk if source and destination are on the same guest
6969                         print "$job: Completing block job...\n";
6970
6971                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6972                         if ($@ =~ m/cannot be completed/) {
6973                             print "$job: Block job cannot be completed, try again.\n";
6974                             $err_complete++;
6975                         }else {
6976                             print "$job: Completed successfully.\n";
6977                             $jobs->{$job}->{complete} = 1;
6978                         }
6979                     }
6980                 }
6981             }
6982             sleep 1;
6983         }
6984     };
6985     my $err = $@;
6986
6987     if ($err) {
6988         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6989         die "mirroring error: $err";
6990     }
6991
6992 }
6993
6994 sub qemu_blockjobs_cancel {
6995     my ($vmid, $jobs) = @_;
6996
6997     foreach my $job (keys %$jobs) {
6998         print "$job: Cancelling block job\n";
6999         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7000         $jobs->{$job}->{cancel} = 1;
7001     }
7002
7003     while (1) {
7004         my $stats = mon_cmd($vmid, "query-block-jobs");
7005
7006         my $running_jobs = {};
7007         foreach my $stat (@$stats) {
7008             $running_jobs->{$stat->{device}} = $stat;
7009         }
7010
7011         foreach my $job (keys %$jobs) {
7012
7013             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7014                 print "$job: Done.\n";
7015                 delete $jobs->{$job};
7016             }
7017         }
7018
7019         last if scalar(keys %$jobs) == 0;
7020
7021         sleep 1;
7022     }
7023 }
7024
7025 sub clone_disk {
7026     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7027         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7028
7029     my $newvolid;
7030
7031     if (!$full) {
7032         print "create linked clone of drive $drivename ($drive->{file})\n";
7033         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7034         push @$newvollist, $newvolid;
7035     } else {
7036
7037         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7038         $storeid = $storage if $storage;
7039
7040         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7041         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7042
7043         print "create full clone of drive $drivename ($drive->{file})\n";
7044         my $name = undef;
7045         if (drive_is_cloudinit($drive)) {
7046             $name = "vm-$newvmid-cloudinit";
7047             $name .= ".$dst_format" if $dst_format ne 'raw';
7048             $snapname = undef;
7049             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7050         }
7051         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7052         push @$newvollist, $newvolid;
7053
7054         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7055
7056         if (drive_is_cloudinit($drive)) {
7057             goto no_data_clone;
7058         }
7059
7060         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7061         if (!$running || $snapname) {
7062             # TODO: handle bwlimits
7063             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7064         } else {
7065
7066             my $kvmver = get_running_qemu_version ($vmid);
7067             if (!min_version($kvmver, 2, 7)) {
7068                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7069                     if $drive->{iothread};
7070             }
7071
7072             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7073         }
7074     }
7075
7076 no_data_clone:
7077     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7078
7079     my $disk = $drive;
7080     $disk->{format} = undef;
7081     $disk->{file} = $newvolid;
7082     $disk->{size} = $size;
7083
7084     return $disk;
7085 }
7086
7087 sub get_running_qemu_version {
7088     my ($vmid) = @_;
7089     my $res = mon_cmd($vmid, "query-version");
7090     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7091 }
7092
7093 sub qemu_use_old_bios_files {
7094     my ($machine_type) = @_;
7095
7096     return if !$machine_type;
7097
7098     my $use_old_bios_files = undef;
7099
7100     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7101         $machine_type = $1;
7102         $use_old_bios_files = 1;
7103     } else {
7104         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7105         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7106         # load new efi bios files on migration. So this hack is required to allow
7107         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7108         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7109         $use_old_bios_files = !min_version($version, 2, 4);
7110     }
7111
7112     return ($use_old_bios_files, $machine_type);
7113 }
7114
7115 sub create_efidisk($$$$$) {
7116     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7117
7118     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7119     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7120
7121     my $vars_size_b = -s $ovmf_vars;
7122     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7123     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7124     PVE::Storage::activate_volumes($storecfg, [$volid]);
7125
7126     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7127     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7128
7129     return ($volid, $size/1024);
7130 }
7131
7132 sub vm_iothreads_list {
7133     my ($vmid) = @_;
7134
7135     my $res = mon_cmd($vmid, 'query-iothreads');
7136
7137     my $iothreads = {};
7138     foreach my $iothread (@$res) {
7139         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7140     }
7141
7142     return $iothreads;
7143 }
7144
7145 sub scsihw_infos {
7146     my ($conf, $drive) = @_;
7147
7148     my $maxdev = 0;
7149
7150     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7151         $maxdev = 7;
7152     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7153         $maxdev = 1;
7154     } else {
7155         $maxdev = 256;
7156     }
7157
7158     my $controller = int($drive->{index} / $maxdev);
7159     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7160
7161     return ($maxdev, $controller, $controller_prefix);
7162 }
7163
7164 sub windows_version {
7165     my ($ostype) = @_;
7166
7167     return 0 if !$ostype;
7168
7169     my $winversion = 0;
7170
7171     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7172         $winversion = 5;
7173     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7174         $winversion = 6;
7175     } elsif ($ostype =~ m/^win(\d+)$/) {
7176         $winversion = $1;
7177     }
7178
7179     return $winversion;
7180 }
7181
7182 sub resolve_dst_disk_format {
7183         my ($storecfg, $storeid, $src_volname, $format) = @_;
7184         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7185
7186         if (!$format) {
7187             # if no target format is specified, use the source disk format as hint
7188             if ($src_volname) {
7189                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7190                 $format = qemu_img_format($scfg, $src_volname);
7191             } else {
7192                 return $defFormat;
7193             }
7194         }
7195
7196         # test if requested format is supported - else use default
7197         my $supported = grep { $_ eq $format } @$validFormats;
7198         $format = $defFormat if !$supported;
7199         return $format;
7200 }
7201
7202 sub resolve_first_disk {
7203     my $conf = shift;
7204     my @disks = PVE::QemuServer::valid_drive_names();
7205     my $firstdisk;
7206     foreach my $ds (reverse @disks) {
7207         next if !$conf->{$ds};
7208         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7209         next if PVE::QemuServer::drive_is_cdrom($disk);
7210         $firstdisk = $ds;
7211     }
7212     return $firstdisk;
7213 }
7214
7215 # NOTE: if this logic changes, please update docs & possibly gui logic
7216 sub find_vmstate_storage {
7217     my ($conf, $storecfg) = @_;
7218
7219     # first, return storage from conf if set
7220     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7221
7222     my ($target, $shared, $local);
7223
7224     foreach_storage_used_by_vm($conf, sub {
7225         my ($sid) = @_;
7226         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7227         my $dst = $scfg->{shared} ? \$shared : \$local;
7228         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7229     });
7230
7231     # second, use shared storage where VM has at least one disk
7232     # third, use local storage where VM has at least one disk
7233     # fall back to local storage
7234     $target = $shared // $local // 'local';
7235
7236     return $target;
7237 }
7238
7239 sub generate_uuid {
7240     my ($uuid, $uuid_str);
7241     UUID::generate($uuid);
7242     UUID::unparse($uuid, $uuid_str);
7243     return $uuid_str;
7244 }
7245
7246 sub generate_smbios1_uuid {
7247     return "uuid=".generate_uuid();
7248 }
7249
7250 sub nbd_stop {
7251     my ($vmid) = @_;
7252
7253     mon_cmd($vmid, 'nbd-server-stop');
7254 }
7255
7256 sub create_reboot_request {
7257     my ($vmid) = @_;
7258     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7259         or die "failed to create reboot trigger file: $!\n";
7260     close($fh);
7261 }
7262
7263 sub clear_reboot_request {
7264     my ($vmid) = @_;
7265     my $path = "/run/qemu-server/$vmid.reboot";
7266     my $res = 0;
7267
7268     $res = unlink($path);
7269     die "could not remove reboot request for $vmid: $!"
7270         if !$res && $! != POSIX::ENOENT;
7271
7272     return $res;
7273 }
7274
7275 # bash completion helper
7276
7277 sub complete_backup_archives {
7278     my ($cmdname, $pname, $cvalue) = @_;
7279
7280     my $cfg = PVE::Storage::config();
7281
7282     my $storeid;
7283
7284     if ($cvalue =~ m/^([^:]+):/) {
7285         $storeid = $1;
7286     }
7287
7288     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7289
7290     my $res = [];
7291     foreach my $id (keys %$data) {
7292         foreach my $item (@{$data->{$id}}) {
7293             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7294             push @$res, $item->{volid} if defined($item->{volid});
7295         }
7296     }
7297
7298     return $res;
7299 }
7300
7301 my $complete_vmid_full = sub {
7302     my ($running) = @_;
7303
7304     my $idlist = vmstatus();
7305
7306     my $res = [];
7307
7308     foreach my $id (keys %$idlist) {
7309         my $d = $idlist->{$id};
7310         if (defined($running)) {
7311             next if $d->{template};
7312             next if $running && $d->{status} ne 'running';
7313             next if !$running && $d->{status} eq 'running';
7314         }
7315         push @$res, $id;
7316
7317     }
7318     return $res;
7319 };
7320
7321 sub complete_vmid {
7322     return &$complete_vmid_full();
7323 }
7324
7325 sub complete_vmid_stopped {
7326     return &$complete_vmid_full(0);
7327 }
7328
7329 sub complete_vmid_running {
7330     return &$complete_vmid_full(1);
7331 }
7332
7333 sub complete_storage {
7334
7335     my $cfg = PVE::Storage::config();
7336     my $ids = $cfg->{ids};
7337
7338     my $res = [];
7339     foreach my $sid (keys %$ids) {
7340         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7341         next if !$ids->{$sid}->{content}->{images};
7342         push @$res, $sid;
7343     }
7344
7345     return $res;
7346 }
7347
7348 sub complete_migration_storage {
7349     my ($cmd, $param, $current_value, $all_args) = @_;
7350
7351     my $targetnode = @$all_args[1];
7352
7353     my $cfg = PVE::Storage::config();
7354     my $ids = $cfg->{ids};
7355
7356     my $res = [];
7357     foreach my $sid (keys %$ids) {
7358         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7359         next if !$ids->{$sid}->{content}->{images};
7360         push @$res, $sid;
7361     }
7362
7363     return $res;
7364 }
7365
7366 1;