PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Machine;
  48 use PVE::QemuServer::Memory;
  49 use PVE::QemuServer::Monitor qw(mon_cmd);
  50 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  51 use PVE::QemuServer::USB qw(parse_usb_device);
  52
  53 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  54 my $OVMF = {
  55     x86_64 => [
  56         "$EDK2_FW_BASE/OVMF_CODE.fd",
  57         "$EDK2_FW_BASE/OVMF_VARS.fd"
  58     ],
  59     aarch64 => [
  60         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  61         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  62     ],
  63 };
  64
  65 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  66
  67 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  68
  69 # Note about locking: we use flock on the config file protect
  70 # against concurent actions.
  71 # Aditionaly, we have a 'lock' setting in the config file. This
  72 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  73 # allowed when such lock is set. But you can ignore this kind of
  74 # lock with the --skiplock flag.
  75
  76 cfs_register_file('/qemu-server/',
  77                   \&parse_vm_config,
  78                   \&write_vm_config);
  79
  80 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  81     description => "Some command save/restore state from this location.",
  82     type => 'string',
  83     maxLength => 128,
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  88     type => 'string',
  89     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  90     description => "The drive's backing file's data format.",
  91     optional => 1,
  92 });
  93
  94 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  95         description => "Specifies the Qemu machine type.",
  96         type => 'string',
  97         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  98         maxLength => 40,
  99         optional => 1,
 100 });
 101
 102 #no warnings 'redefine';
 103
 104 sub cgroups_write {
 105    my ($controller, $vmid, $option, $value) = @_;
 106
 107    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 108    PVE::ProcFSTools::write_proc_entry($path, $value);
 109
 110 }
 111
 112 my $nodename_cache;
 113 sub nodename {
 114     $nodename_cache //= PVE::INotify::nodename();
 115     return $nodename_cache;
 116 }
 117
 118 my $watchdog_fmt = {
 119     model => {
 120         default_key => 1,
 121         type => 'string',
 122         enum => [qw(i6300esb ib700)],
 123         description => "Watchdog type to emulate.",
 124         default => 'i6300esb',
 125         optional => 1,
 126     },
 127     action => {
 128         type => 'string',
 129         enum => [qw(reset shutdown poweroff pause debug none)],
 130         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 131         optional => 1,
 132     },
 133 };
 134 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 135
 136 my $agent_fmt = {
 137     enabled => {
 138         description => "Enable/disable Qemu GuestAgent.",
 139         type => 'boolean',
 140         default => 0,
 141         default_key => 1,
 142     },
 143     fstrim_cloned_disks => {
 144         description => "Run fstrim after cloning/moving a disk.",
 145         type => 'boolean',
 146         optional => 1,
 147         default => 0
 148     },
 149     type => {
 150         description => "Select the agent type",
 151         type => 'string',
 152         default => 'virtio',
 153         optional => 1,
 154         enum => [qw(virtio isa)],
 155     },
 156 };
 157
 158 my $vga_fmt = {
 159     type => {
 160         description => "Select the VGA type.",
 161         type => 'string',
 162         default => 'std',
 163         optional => 1,
 164         default_key => 1,
 165         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 166     },
 167     memory => {
 168         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 169         type => 'integer',
 170         optional => 1,
 171         minimum => 4,
 172         maximum => 512,
 173     },
 174 };
 175
 176 my $ivshmem_fmt = {
 177     size => {
 178         type => 'integer',
 179         minimum => 1,
 180         description => "The size of the file in MB.",
 181     },
 182     name => {
 183         type => 'string',
 184         pattern => '[a-zA-Z0-9\-]+',
 185         optional => 1,
 186         format_description => 'string',
 187         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 188     },
 189 };
 190
 191 my $audio_fmt = {
 192     device => {
 193         type => 'string',
 194         enum => [qw(ich9-intel-hda intel-hda AC97)],
 195         description =>  "Configure an audio device."
 196     },
 197     driver =>  {
 198         type => 'string',
 199         enum => ['spice'],
 200         default => 'spice',
 201         optional => 1,
 202         description => "Driver backend for the audio device."
 203     },
 204 };
 205
 206 my $spice_enhancements_fmt = {
 207     foldersharing => {
 208         type => 'boolean',
 209         optional => 1,
 210         default => '0',
 211         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 212     },
 213     videostreaming =>  {
 214         type => 'string',
 215         enum => ['off', 'all', 'filter'],
 216         default => 'off',
 217         optional => 1,
 218         description => "Enable video streaming. Uses compression for detected video streams."
 219     },
 220 };
 221
 222 my $confdesc = {
 223     onboot => {
 224         optional => 1,
 225         type => 'boolean',
 226         description => "Specifies whether a VM will be started during system bootup.",
 227         default => 0,
 228     },
 229     autostart => {
 230         optional => 1,
 231         type => 'boolean',
 232         description => "Automatic restart after crash (currently ignored).",
 233         default => 0,
 234     },
 235     hotplug => {
 236         optional => 1,
 237         type => 'string', format => 'pve-hotplug-features',
 238         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 239         default => 'network,disk,usb',
 240     },
 241     reboot => {
 242         optional => 1,
 243         type => 'boolean',
 244         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 245         default => 1,
 246     },
 247     lock => {
 248         optional => 1,
 249         type => 'string',
 250         description => "Lock/unlock the VM.",
 251         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 252     },
 253     cpulimit => {
 254         optional => 1,
 255         type => 'number',
 256         description => "Limit of CPU usage.",
 257         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 258         minimum => 0,
 259         maximum => 128,
 260         default => 0,
 261     },
 262     cpuunits => {
 263         optional => 1,
 264         type => 'integer',
 265         description => "CPU weight for a VM.",
 266         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 267         minimum => 2,
 268         maximum => 262144,
 269         default => 1024,
 270     },
 271     memory => {
 272         optional => 1,
 273         type => 'integer',
 274         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 275         minimum => 16,
 276         default => 512,
 277     },
 278     balloon => {
 279         optional => 1,
 280         type => 'integer',
 281         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 282         minimum => 0,
 283     },
 284     shares => {
 285         optional => 1,
 286         type => 'integer',
 287         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 288         minimum => 0,
 289         maximum => 50000,
 290         default => 1000,
 291     },
 292     keyboard => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 296                        "It should not be necessary to set it.",
 297         enum => PVE::Tools::kvmkeymaplist(),
 298         default => undef,
 299     },
 300     name => {
 301         optional => 1,
 302         type => 'string', format => 'dns-name',
 303         description => "Set a name for the VM. Only used on the configuration web interface.",
 304     },
 305     scsihw => {
 306         optional => 1,
 307         type => 'string',
 308         description => "SCSI controller model",
 309         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 310         default => 'lsi',
 311     },
 312     description => {
 313         optional => 1,
 314         type => 'string',
 315         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 316     },
 317     ostype => {
 318         optional => 1,
 319         type => 'string',
 320         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 321         description => "Specify guest operating system.",
 322         verbose_description => <<EODESC,
 323 Specify guest operating system. This is used to enable special
 324 optimization/features for specific operating systems:
 325
 326 [horizontal]
 327 other;; unspecified OS
 328 wxp;; Microsoft Windows XP
 329 w2k;; Microsoft Windows 2000
 330 w2k3;; Microsoft Windows 2003
 331 w2k8;; Microsoft Windows 2008
 332 wvista;; Microsoft Windows Vista
 333 win7;; Microsoft Windows 7
 334 win8;; Microsoft Windows 8/2012/2012r2
 335 win10;; Microsoft Windows 10/2016
 336 l24;; Linux 2.4 Kernel
 337 l26;; Linux 2.6 - 5.X Kernel
 338 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 339 EODESC
 340     },
 341     boot => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 345         pattern => '[acdn]{1,4}',
 346         default => 'cdn',
 347     },
 348     bootdisk => {
 349         optional => 1,
 350         type => 'string', format => 'pve-qm-bootdisk',
 351         description => "Enable booting from specified disk.",
 352         pattern => '(ide|sata|scsi|virtio)\d+',
 353     },
 354     smp => {
 355         optional => 1,
 356         type => 'integer',
 357         description => "The number of CPUs. Please use option -sockets instead.",
 358         minimum => 1,
 359         default => 1,
 360     },
 361     sockets => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "The number of CPU sockets.",
 365         minimum => 1,
 366         default => 1,
 367     },
 368     cores => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "The number of cores per socket.",
 372         minimum => 1,
 373         default => 1,
 374     },
 375     numa => {
 376         optional => 1,
 377         type => 'boolean',
 378         description => "Enable/disable NUMA.",
 379         default => 0,
 380     },
 381     hugepages => {
 382         optional => 1,
 383         type => 'string',
 384         description => "Enable/disable hugepages memory.",
 385         enum => [qw(any 2 1024)],
 386     },
 387     vcpus => {
 388         optional => 1,
 389         type => 'integer',
 390         description => "Number of hotplugged vcpus.",
 391         minimum => 1,
 392         default => 0,
 393     },
 394     acpi => {
 395         optional => 1,
 396         type => 'boolean',
 397         description => "Enable/disable ACPI.",
 398         default => 1,
 399     },
 400     agent => {
 401         optional => 1,
 402         description => "Enable/disable Qemu GuestAgent and its properties.",
 403         type => 'string',
 404         format => $agent_fmt,
 405     },
 406     kvm => {
 407         optional => 1,
 408         type => 'boolean',
 409         description => "Enable/disable KVM hardware virtualization.",
 410         default => 1,
 411     },
 412     tdf => {
 413         optional => 1,
 414         type => 'boolean',
 415         description => "Enable/disable time drift fix.",
 416         default => 0,
 417     },
 418     localtime => {
 419         optional => 1,
 420         type => 'boolean',
 421         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 422     },
 423     freeze => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 427     },
 428     vga => {
 429         optional => 1,
 430         type => 'string', format => $vga_fmt,
 431         description => "Configure the VGA hardware.",
 432         verbose_description => "Configure the VGA Hardware. If you want to use ".
 433             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 434             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 435             "is 'std' for all OS types besides some Windows versions (XP and " .
 436             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 437             "display server. For win* OS you can select how many independent " .
 438             "displays you want, Linux guests can add displays them self.\n".
 439             "You can also run without any graphic card, using a serial device as terminal.",
 440     },
 441     watchdog => {
 442         optional => 1,
 443         type => 'string', format => 'pve-qm-watchdog',
 444         description => "Create a virtual hardware watchdog device.",
 445         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 446             " (by a guest action), the watchdog must be periodically polled " .
 447             "by an agent inside the guest or else the watchdog will reset " .
 448             "the guest (or execute the respective action specified)",
 449     },
 450     startdate => {
 451         optional => 1,
 452         type => 'string',
 453         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 454         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 455         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 456         default => 'now',
 457     },
 458     startup =>  get_standard_option('pve-startup-order'),
 459     template => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Enable/disable Template.",
 463         default => 0,
 464     },
 465     args => {
 466         optional => 1,
 467         type => 'string',
 468         description => "Arbitrary arguments passed to kvm.",
 469         verbose_description => <<EODESCR,
 470 Arbitrary arguments passed to kvm, for example:
 471
 472 args: -no-reboot -no-hpet
 473
 474 NOTE: this option is for experts only.
 475 EODESCR
 476     },
 477     tablet => {
 478         optional => 1,
 479         type => 'boolean',
 480         default => 1,
 481         description => "Enable/disable the USB tablet device.",
 482         verbose_description => "Enable/disable the USB tablet device. This device is " .
 483             "usually needed to allow absolute mouse positioning with VNC. " .
 484             "Else the mouse runs out of sync with normal VNC clients. " .
 485             "If you're running lots of console-only guests on one host, " .
 486             "you may consider disabling this to save some context switches. " .
 487             "This is turned off by default if you use spice (-vga=qxl).",
 488     },
 489     migrate_speed => {
 490         optional => 1,
 491         type => 'integer',
 492         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 493         minimum => 0,
 494         default => 0,
 495     },
 496     migrate_downtime => {
 497         optional => 1,
 498         type => 'number',
 499         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 500         minimum => 0,
 501         default => 0.1,
 502     },
 503     cdrom => {
 504         optional => 1,
 505         type => 'string', format => 'pve-qm-ide',
 506         typetext => '<volume>',
 507         description => "This is an alias for option -ide2",
 508     },
 509     cpu => {
 510         optional => 1,
 511         description => "Emulated CPU type.",
 512         type => 'string',
 513         format => $PVE::QemuServer::CPUConfig::cpu_fmt,
 514     },
 515     parent => get_standard_option('pve-snapshot-name', {
 516         optional => 1,
 517         description => "Parent snapshot name. This is used internally, and should not be modified.",
 518     }),
 519     snaptime => {
 520         optional => 1,
 521         description => "Timestamp for snapshots.",
 522         type => 'integer',
 523         minimum => 0,
 524     },
 525     vmstate => {
 526         optional => 1,
 527         type => 'string', format => 'pve-volume-id',
 528         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 529     },
 530     vmstatestorage => get_standard_option('pve-storage-id', {
 531         description => "Default storage for VM state volumes/files.",
 532         optional => 1,
 533     }),
 534     runningmachine => get_standard_option('pve-qemu-machine', {
 535         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 536     }),
 537     machine => get_standard_option('pve-qemu-machine'),
 538     arch => {
 539         description => "Virtual processor architecture. Defaults to the host.",
 540         optional => 1,
 541         type => 'string',
 542         enum => [qw(x86_64 aarch64)],
 543     },
 544     smbios1 => {
 545         description => "Specify SMBIOS type 1 fields.",
 546         type => 'string', format => 'pve-qm-smbios1',
 547         maxLength => 512,
 548         optional => 1,
 549     },
 550     protection => {
 551         optional => 1,
 552         type => 'boolean',
 553         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 554         default => 0,
 555     },
 556     bios => {
 557         optional => 1,
 558         type => 'string',
 559         enum => [ qw(seabios ovmf) ],
 560         description => "Select BIOS implementation.",
 561         default => 'seabios',
 562     },
 563     vmgenid => {
 564         type => 'string',
 565         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 566         format_description => 'UUID',
 567         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 568         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 569             " 128-bit integer value identifier to the guest OS. This allows to".
 570             " notify the guest operating system when the virtual machine is".
 571             " executed with a different configuration (e.g. snapshot execution".
 572             " or creation from a template). The guest operating system notices".
 573             " the change, and is then able to react as appropriate by marking".
 574             " its copies of distributed databases as dirty, re-initializing its".
 575             " random number generator, etc.\n".
 576             "Note that auto-creation only works when done throug API/CLI create".
 577             " or update methods, but not when manually editing the config file.",
 578         default => "1 (autogenerated)",
 579         optional => 1,
 580     },
 581     hookscript => {
 582         type => 'string',
 583         format => 'pve-volume-id',
 584         optional => 1,
 585         description => "Script that will be executed during various steps in the vms lifetime.",
 586     },
 587     ivshmem => {
 588         type => 'string',
 589         format => $ivshmem_fmt,
 590         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 591         optional => 1,
 592     },
 593     audio0 => {
 594         type => 'string',
 595         format => $audio_fmt,
 596         description => "Configure a audio device, useful in combination with QXL/Spice.",
 597         optional => 1
 598     },
 599     spice_enhancements => {
 600         type => 'string',
 601         format => $spice_enhancements_fmt,
 602         description => "Configure additional enhancements for SPICE.",
 603         optional => 1
 604     },
 605     tags => {
 606         type => 'string', format => 'pve-tag-list',
 607         description => 'Tags of the VM. This is only meta information.',
 608         optional => 1,
 609     },
 610 };
 611
 612 my $cicustom_fmt = {
 613     meta => {
 614         type => 'string',
 615         optional => 1,
 616         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 617         format => 'pve-volume-id',
 618         format_description => 'volume',
 619     },
 620     network => {
 621         type => 'string',
 622         optional => 1,
 623         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 624         format => 'pve-volume-id',
 625         format_description => 'volume',
 626     },
 627     user => {
 628         type => 'string',
 629         optional => 1,
 630         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 631         format => 'pve-volume-id',
 632         format_description => 'volume',
 633     },
 634 };
 635 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 636
 637 my $confdesc_cloudinit = {
 638     citype => {
 639         optional => 1,
 640         type => 'string',
 641         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 642         enum => ['configdrive2', 'nocloud'],
 643     },
 644     ciuser => {
 645         optional => 1,
 646         type => 'string',
 647         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 648     },
 649     cipassword => {
 650         optional => 1,
 651         type => 'string',
 652         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 653     },
 654     cicustom => {
 655         optional => 1,
 656         type => 'string',
 657         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 658         format => 'pve-qm-cicustom',
 659     },
 660     searchdomain => {
 661         optional => 1,
 662         type => 'string',
 663         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 664     },
 665     nameserver => {
 666         optional => 1,
 667         type => 'string', format => 'address-list',
 668         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 669     },
 670     sshkeys => {
 671         optional => 1,
 672         type => 'string',
 673         format => 'urlencoded',
 674         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 675     },
 676 };
 677
 678 # what about other qemu settings ?
 679 #cpu => 'string',
 680 #machine => 'string',
 681 #fda => 'file',
 682 #fdb => 'file',
 683 #mtdblock => 'file',
 684 #sd => 'file',
 685 #pflash => 'file',
 686 #snapshot => 'bool',
 687 #bootp => 'file',
 688 ##tftp => 'dir',
 689 ##smb => 'dir',
 690 #kernel => 'file',
 691 #append => 'string',
 692 #initrd => 'file',
 693 ##soundhw => 'string',
 694
 695 while (my ($k, $v) = each %$confdesc) {
 696     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 697 }
 698
 699 my $MAX_IDE_DISKS = 4;
 700 my $MAX_SCSI_DISKS = 14;
 701 my $MAX_VIRTIO_DISKS = 16;
 702 my $MAX_SATA_DISKS = 6;
 703 my $MAX_USB_DEVICES = 5;
 704 my $MAX_NETS = 32;
 705 my $MAX_UNUSED_DISKS = 256;
 706 my $MAX_HOSTPCI_DEVICES = 16;
 707 my $MAX_SERIAL_PORTS = 4;
 708 my $MAX_PARALLEL_PORTS = 3;
 709 my $MAX_NUMA = 8;
 710
 711 my $numa_fmt = {
 712     cpus => {
 713         type => "string",
 714         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 715         description => "CPUs accessing this NUMA node.",
 716         format_description => "id[-id];...",
 717     },
 718     memory => {
 719         type => "number",
 720         description => "Amount of memory this NUMA node provides.",
 721         optional => 1,
 722     },
 723     hostnodes => {
 724         type => "string",
 725         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 726         description => "Host NUMA nodes to use.",
 727         format_description => "id[-id];...",
 728         optional => 1,
 729     },
 730     policy => {
 731         type => 'string',
 732         enum => [qw(preferred bind interleave)],
 733         description => "NUMA allocation policy.",
 734         optional => 1,
 735     },
 736 };
 737 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 738 my $numadesc = {
 739     optional => 1,
 740     type => 'string', format => $numa_fmt,
 741     description => "NUMA topology.",
 742 };
 743 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 744
 745 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 746     $confdesc->{"numa$i"} = $numadesc;
 747 }
 748
 749 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 750                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 751                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 752 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 753
 754 my $net_fmt_bridge_descr = <<__EOD__;
 755 Bridge to attach the network device to. The Proxmox VE standard bridge
 756 is called 'vmbr0'.
 757
 758 If you do not specify a bridge, we create a kvm user (NATed) network
 759 device, which provides DHCP and DNS services. The following addresses
 760 are used:
 761
 762  10.0.2.2   Gateway
 763  10.0.2.3   DNS Server
 764  10.0.2.4   SMB Server
 765
 766 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 767 __EOD__
 768
 769 my $net_fmt = {
 770     macaddr  => get_standard_option('mac-addr', {
 771         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 772     }),
 773     model => {
 774         type => 'string',
 775         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 776         enum => $nic_model_list,
 777         default_key => 1,
 778     },
 779     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 780     bridge => {
 781         type => 'string',
 782         description => $net_fmt_bridge_descr,
 783         format_description => 'bridge',
 784         optional => 1,
 785     },
 786     queues => {
 787         type => 'integer',
 788         minimum => 0, maximum => 16,
 789         description => 'Number of packet queues to be used on the device.',
 790         optional => 1,
 791     },
 792     rate => {
 793         type => 'number',
 794         minimum => 0,
 795         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 796         optional => 1,
 797     },
 798     tag => {
 799         type => 'integer',
 800         minimum => 1, maximum => 4094,
 801         description => 'VLAN tag to apply to packets on this interface.',
 802         optional => 1,
 803     },
 804     trunks => {
 805         type => 'string',
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => 'VLAN trunks to pass through this interface.',
 808         format_description => 'vlanid[;vlanid...]',
 809         optional => 1,
 810     },
 811     firewall => {
 812         type => 'boolean',
 813         description => 'Whether this interface should be protected by the firewall.',
 814         optional => 1,
 815     },
 816     link_down => {
 817         type => 'boolean',
 818         description => 'Whether this interface should be disconnected (like pulling the plug).',
 819         optional => 1,
 820     },
 821 };
 822
 823 my $netdesc = {
 824     optional => 1,
 825     type => 'string', format => $net_fmt,
 826     description => "Specify network devices.",
 827 };
 828
 829 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 830
 831 my $ipconfig_fmt = {
 832     ip => {
 833         type => 'string',
 834         format => 'pve-ipv4-config',
 835         format_description => 'IPv4Format/CIDR',
 836         description => 'IPv4 address in CIDR format.',
 837         optional => 1,
 838         default => 'dhcp',
 839     },
 840     gw => {
 841         type => 'string',
 842         format => 'ipv4',
 843         format_description => 'GatewayIPv4',
 844         description => 'Default gateway for IPv4 traffic.',
 845         optional => 1,
 846         requires => 'ip',
 847     },
 848     ip6 => {
 849         type => 'string',
 850         format => 'pve-ipv6-config',
 851         format_description => 'IPv6Format/CIDR',
 852         description => 'IPv6 address in CIDR format.',
 853         optional => 1,
 854         default => 'dhcp',
 855     },
 856     gw6 => {
 857         type => 'string',
 858         format => 'ipv6',
 859         format_description => 'GatewayIPv6',
 860         description => 'Default gateway for IPv6 traffic.',
 861         optional => 1,
 862         requires => 'ip6',
 863     },
 864 };
 865 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 866 my $ipconfigdesc = {
 867     optional => 1,
 868     type => 'string', format => 'pve-qm-ipconfig',
 869     description => <<'EODESCR',
 870 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 871
 872 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 873
 874 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 875 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 876
 877 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 878 EODESCR
 879 };
 880 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 881
 882 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 883     $confdesc->{"net$i"} = $netdesc;
 884     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 885 }
 886
 887 foreach my $key (keys %$confdesc_cloudinit) {
 888     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 889 }
 890
 891 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 892 sub verify_volume_id_or_qm_path {
 893     my ($volid, $noerr) = @_;
 894
 895     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 896         return $volid;
 897     }
 898
 899     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 900     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 901     if ($@) {
 902         return undef if $noerr;
 903         die $@;
 904     }
 905     return $volid;
 906 }
 907
 908 my $drivename_hash;
 909
 910 my %drivedesc_base = (
 911     volume => { alias => 'file' },
 912     file => {
 913         type => 'string',
 914         format => 'pve-volume-id-or-qm-path',
 915         default_key => 1,
 916         format_description => 'volume',
 917         description => "The drive's backing volume.",
 918     },
 919     media => {
 920         type => 'string',
 921         enum => [qw(cdrom disk)],
 922         description => "The drive's media type.",
 923         default => 'disk',
 924         optional => 1
 925     },
 926     cyls => {
 927         type => 'integer',
 928         description => "Force the drive's physical geometry to have a specific cylinder count.",
 929         optional => 1
 930     },
 931     heads => {
 932         type => 'integer',
 933         description => "Force the drive's physical geometry to have a specific head count.",
 934         optional => 1
 935     },
 936     secs => {
 937         type => 'integer',
 938         description => "Force the drive's physical geometry to have a specific sector count.",
 939         optional => 1
 940     },
 941     trans => {
 942         type => 'string',
 943         enum => [qw(none lba auto)],
 944         description => "Force disk geometry bios translation mode.",
 945         optional => 1,
 946     },
 947     snapshot => {
 948         type => 'boolean',
 949         description => "Controls qemu's snapshot mode feature."
 950             . " If activated, changes made to the disk are temporary and will"
 951             . " be discarded when the VM is shutdown.",
 952         optional => 1,
 953     },
 954     cache => {
 955         type => 'string',
 956         enum => [qw(none writethrough writeback unsafe directsync)],
 957         description => "The drive's cache mode",
 958         optional => 1,
 959     },
 960     format => get_standard_option('pve-qm-image-format'),
 961     size => {
 962         type => 'string',
 963         format => 'disk-size',
 964         format_description => 'DiskSize',
 965         description => "Disk size. This is purely informational and has no effect.",
 966         optional => 1,
 967     },
 968     backup => {
 969         type => 'boolean',
 970         description => "Whether the drive should be included when making backups.",
 971         optional => 1,
 972     },
 973     replicate => {
 974         type => 'boolean',
 975         description => 'Whether the drive should considered for replication jobs.',
 976         optional => 1,
 977         default => 1,
 978     },
 979     rerror => {
 980         type => 'string',
 981         enum => [qw(ignore report stop)],
 982         description => 'Read error action.',
 983         optional => 1,
 984     },
 985     werror => {
 986         type => 'string',
 987         enum => [qw(enospc ignore report stop)],
 988         description => 'Write error action.',
 989         optional => 1,
 990     },
 991     aio => {
 992         type => 'string',
 993         enum => [qw(native threads)],
 994         description => 'AIO type to use.',
 995         optional => 1,
 996     },
 997     discard => {
 998         type => 'string',
 999         enum => [qw(ignore on)],
1000         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1001         optional => 1,
1002     },
1003     detect_zeroes => {
1004         type => 'boolean',
1005         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1006         optional => 1,
1007     },
1008     serial => {
1009         type => 'string',
1010         format => 'urlencoded',
1011         format_description => 'serial',
1012         maxLength => 20*3, # *3 since it's %xx url enoded
1013         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1014         optional => 1,
1015     },
1016     shared => {
1017         type => 'boolean',
1018         description => 'Mark this locally-managed volume as available on all nodes',
1019         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1020         optional => 1,
1021         default => 0,
1022     }
1023 );
1024
1025 my %iothread_fmt = ( iothread => {
1026         type => 'boolean',
1027         description => "Whether to use iothreads for this drive",
1028         optional => 1,
1029 });
1030
1031 my %model_fmt = (
1032     model => {
1033         type => 'string',
1034         format => 'urlencoded',
1035         format_description => 'model',
1036         maxLength => 40*3, # *3 since it's %xx url enoded
1037         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1038         optional => 1,
1039     },
1040 );
1041
1042 my %queues_fmt = (
1043     queues => {
1044         type => 'integer',
1045         description => "Number of queues.",
1046         minimum => 2,
1047         optional => 1
1048     }
1049 );
1050
1051 my %scsiblock_fmt = (
1052     scsiblock => {
1053         type => 'boolean',
1054         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1055         optional => 1,
1056         default => 0,
1057     },
1058 );
1059
1060 my %ssd_fmt = (
1061     ssd => {
1062         type => 'boolean',
1063         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1064         optional => 1,
1065     },
1066 );
1067
1068 my %wwn_fmt = (
1069     wwn => {
1070         type => 'string',
1071         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1072         format_description => 'wwn',
1073         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1074         optional => 1,
1075     },
1076 );
1077
1078 my $add_throttle_desc = sub {
1079     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1080     my $d = {
1081         type => $type,
1082         format_description => $unit,
1083         description => "Maximum $what in $longunit.",
1084         optional => 1,
1085     };
1086     $d->{minimum} = $minimum if defined($minimum);
1087     $drivedesc_base{$key} = $d;
1088 };
1089 # throughput: (leaky bucket)
1090 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1091 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1092 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1093 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1094 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1095 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1096 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1097 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1098 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1099
1100 # pools: (pool of IO before throttling starts taking effect)
1101 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1102 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1103 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1104 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1105 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1106 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1107
1108 # burst lengths
1109 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1110 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1111 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1112 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1113 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1114 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1115
1116 # legacy support
1117 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1118 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1119 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1120 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1121
1122 my $ide_fmt = {
1123     %drivedesc_base,
1124     %model_fmt,
1125     %ssd_fmt,
1126     %wwn_fmt,
1127 };
1128 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1129
1130 my $idedesc = {
1131     optional => 1,
1132     type => 'string', format => $ide_fmt,
1133     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1134 };
1135 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1136
1137 my $scsi_fmt = {
1138     %drivedesc_base,
1139     %iothread_fmt,
1140     %queues_fmt,
1141     %scsiblock_fmt,
1142     %ssd_fmt,
1143     %wwn_fmt,
1144 };
1145 my $scsidesc = {
1146     optional => 1,
1147     type => 'string', format => $scsi_fmt,
1148     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1149 };
1150 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1151
1152 my $sata_fmt = {
1153     %drivedesc_base,
1154     %ssd_fmt,
1155     %wwn_fmt,
1156 };
1157 my $satadesc = {
1158     optional => 1,
1159     type => 'string', format => $sata_fmt,
1160     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1161 };
1162 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1163
1164 my $virtio_fmt = {
1165     %drivedesc_base,
1166     %iothread_fmt,
1167 };
1168 my $virtiodesc = {
1169     optional => 1,
1170     type => 'string', format => $virtio_fmt,
1171     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1172 };
1173 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1174
1175 my $alldrive_fmt = {
1176     %drivedesc_base,
1177     %iothread_fmt,
1178     %model_fmt,
1179     %queues_fmt,
1180     %scsiblock_fmt,
1181     %ssd_fmt,
1182     %wwn_fmt,
1183 };
1184
1185 my $efidisk_fmt = {
1186     volume => { alias => 'file' },
1187     file => {
1188         type => 'string',
1189         format => 'pve-volume-id-or-qm-path',
1190         default_key => 1,
1191         format_description => 'volume',
1192         description => "The drive's backing volume.",
1193     },
1194     format => get_standard_option('pve-qm-image-format'),
1195     size => {
1196         type => 'string',
1197         format => 'disk-size',
1198         format_description => 'DiskSize',
1199         description => "Disk size. This is purely informational and has no effect.",
1200         optional => 1,
1201     },
1202 };
1203
1204 my $efidisk_desc = {
1205     optional => 1,
1206     type => 'string', format => $efidisk_fmt,
1207     description => "Configure a Disk for storing EFI vars",
1208 };
1209
1210 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1211
1212 my $usb_fmt = {
1213     host => {
1214         default_key => 1,
1215         type => 'string', format => 'pve-qm-usb-device',
1216         format_description => 'HOSTUSBDEVICE|spice',
1217         description => <<EODESCR,
1218 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1219
1220  'bus-port(.port)*' (decimal numbers) or
1221  'vendor_id:product_id' (hexadeciaml numbers) or
1222  'spice'
1223
1224 You can use the 'lsusb -t' command to list existing usb devices.
1225
1226 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1227
1228 The value 'spice' can be used to add a usb redirection devices for spice.
1229 EODESCR
1230     },
1231     usb3 => {
1232         optional => 1,
1233         type => 'boolean',
1234         description => "Specifies whether if given host option is a USB3 device or port.",
1235         default => 0,
1236     },
1237 };
1238
1239 my $usbdesc = {
1240     optional => 1,
1241     type => 'string', format => $usb_fmt,
1242     description => "Configure an USB device (n is 0 to 4).",
1243 };
1244 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1245
1246 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1247 my $hostpci_fmt = {
1248     host => {
1249         default_key => 1,
1250         type => 'string',
1251         pattern => qr/$PCIRE(;$PCIRE)*/,
1252         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1253         description => <<EODESCR,
1254 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1255 of PCI virtual functions of the host. HOSTPCIID syntax is:
1256
1257 'bus:dev.func' (hexadecimal numbers)
1258
1259 You can us the 'lspci' command to list existing PCI devices.
1260 EODESCR
1261     },
1262     rombar => {
1263         type => 'boolean',
1264         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1265         optional => 1,
1266         default => 1,
1267     },
1268     romfile => {
1269         type => 'string',
1270         pattern => '[^,;]+',
1271         format_description => 'string',
1272         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1273         optional => 1,
1274     },
1275     pcie => {
1276         type => 'boolean',
1277         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1278         optional => 1,
1279         default => 0,
1280     },
1281     'x-vga' => {
1282         type => 'boolean',
1283         description =>  "Enable vfio-vga device support.",
1284         optional => 1,
1285         default => 0,
1286     },
1287     'mdev' => {
1288         type => 'string',
1289         format_description => 'string',
1290         pattern => '[^/\.:]+',
1291         optional => 1,
1292         description => <<EODESCR
1293 The type of mediated device to use.
1294 An instance of this type will be created on startup of the VM and
1295 will be cleaned up when the VM stops.
1296 EODESCR
1297     }
1298 };
1299 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1300
1301 my $hostpcidesc = {
1302         optional => 1,
1303         type => 'string', format => 'pve-qm-hostpci',
1304         description => "Map host PCI devices into guest.",
1305         verbose_description =>  <<EODESCR,
1306 Map host PCI devices into guest.
1307
1308 NOTE: This option allows direct access to host hardware. So it is no longer
1309 possible to migrate such machines - use with special care.
1310
1311 CAUTION: Experimental! User reported problems with this option.
1312 EODESCR
1313 };
1314 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1315
1316 my $serialdesc = {
1317         optional => 1,
1318         type => 'string',
1319         pattern => '(/dev/.+|socket)',
1320         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1321         verbose_description =>  <<EODESCR,
1322 Create a serial device inside the VM (n is 0 to 3), and pass through a
1323 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1324 host side (use 'qm terminal' to open a terminal connection).
1325
1326 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1327
1328 CAUTION: Experimental! User reported problems with this option.
1329 EODESCR
1330 };
1331
1332 my $paralleldesc= {
1333         optional => 1,
1334         type => 'string',
1335         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1336         description =>  "Map host parallel devices (n is 0 to 2).",
1337         verbose_description =>  <<EODESCR,
1338 Map host parallel devices (n is 0 to 2).
1339
1340 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1341
1342 CAUTION: Experimental! User reported problems with this option.
1343 EODESCR
1344 };
1345
1346 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1347     $confdesc->{"parallel$i"} = $paralleldesc;
1348 }
1349
1350 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1351     $confdesc->{"serial$i"} = $serialdesc;
1352 }
1353
1354 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1355     $confdesc->{"hostpci$i"} = $hostpcidesc;
1356 }
1357
1358 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1359     $drivename_hash->{"ide$i"} = 1;
1360     $confdesc->{"ide$i"} = $idedesc;
1361 }
1362
1363 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1364     $drivename_hash->{"sata$i"} = 1;
1365     $confdesc->{"sata$i"} = $satadesc;
1366 }
1367
1368 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1369     $drivename_hash->{"scsi$i"} = 1;
1370     $confdesc->{"scsi$i"} = $scsidesc ;
1371 }
1372
1373 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1374     $drivename_hash->{"virtio$i"} = 1;
1375     $confdesc->{"virtio$i"} = $virtiodesc;
1376 }
1377
1378 $drivename_hash->{efidisk0} = 1;
1379 $confdesc->{efidisk0} = $efidisk_desc;
1380
1381 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1382     $confdesc->{"usb$i"} = $usbdesc;
1383 }
1384
1385 my $unuseddesc = {
1386     optional => 1,
1387     type => 'string', format => 'pve-volume-id',
1388     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1389 };
1390
1391 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1392     $confdesc->{"unused$i"} = $unuseddesc;
1393 }
1394
1395 my $kvm_api_version = 0;
1396
1397 sub kvm_version {
1398     return $kvm_api_version if $kvm_api_version;
1399
1400     open my $fh, '<', '/dev/kvm'
1401         or return undef;
1402
1403     # 0xae00 => KVM_GET_API_VERSION
1404     $kvm_api_version = ioctl($fh, 0xae00, 0);
1405
1406     return $kvm_api_version;
1407 }
1408
1409 my $kvm_user_version = {};
1410 my $kvm_mtime = {};
1411
1412 sub kvm_user_version {
1413     my ($binary) = @_;
1414
1415     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1416     my $st = stat($binary);
1417
1418     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1419     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1420         $cachedmtime == $st->mtime;
1421
1422     $kvm_user_version->{$binary} = 'unknown';
1423     $kvm_mtime->{$binary} = $st->mtime;
1424
1425     my $code = sub {
1426         my $line = shift;
1427         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1428             $kvm_user_version->{$binary} = $2;
1429         }
1430     };
1431
1432     eval { run_command([$binary, '--version'], outfunc => $code); };
1433     warn $@ if $@;
1434
1435     return $kvm_user_version->{$binary};
1436
1437 }
1438
1439 sub kernel_has_vhost_net {
1440     return -c '/dev/vhost-net';
1441 }
1442
1443 sub valid_drive_names {
1444     # order is important - used to autoselect boot disk
1445     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1446             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1447             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1448             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1449             'efidisk0');
1450 }
1451
1452 sub is_valid_drivename {
1453     my $dev = shift;
1454
1455     return defined($drivename_hash->{$dev});
1456 }
1457
1458 sub option_exists {
1459     my $key = shift;
1460     return defined($confdesc->{$key});
1461 }
1462
1463 my $cdrom_path;
1464 sub get_cdrom_path {
1465
1466     return  $cdrom_path if $cdrom_path;
1467
1468     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1469     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1470     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1471 }
1472
1473 sub get_iso_path {
1474     my ($storecfg, $vmid, $cdrom) = @_;
1475
1476     if ($cdrom eq 'cdrom') {
1477         return get_cdrom_path();
1478     } elsif ($cdrom eq 'none') {
1479         return '';
1480     } elsif ($cdrom =~ m|^/|) {
1481         return $cdrom;
1482     } else {
1483         return PVE::Storage::path($storecfg, $cdrom);
1484     }
1485 }
1486
1487 # try to convert old style file names to volume IDs
1488 sub filename_to_volume_id {
1489     my ($vmid, $file, $media) = @_;
1490
1491      if (!($file eq 'none' || $file eq 'cdrom' ||
1492           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1493
1494         return undef if $file =~ m|/|;
1495
1496         if ($media && $media eq 'cdrom') {
1497             $file = "local:iso/$file";
1498         } else {
1499             $file = "local:$vmid/$file";
1500         }
1501     }
1502
1503     return $file;
1504 }
1505
1506 sub verify_media_type {
1507     my ($opt, $vtype, $media) = @_;
1508
1509     return if !$media;
1510
1511     my $etype;
1512     if ($media eq 'disk') {
1513         $etype = 'images';
1514     } elsif ($media eq 'cdrom') {
1515         $etype = 'iso';
1516     } else {
1517         die "internal error";
1518     }
1519
1520     return if ($vtype eq $etype);
1521
1522     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1523 }
1524
1525 sub cleanup_drive_path {
1526     my ($opt, $storecfg, $drive) = @_;
1527
1528     # try to convert filesystem paths to volume IDs
1529
1530     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1531         ($drive->{file} !~ m|^/dev/.+|) &&
1532         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1533         ($drive->{file} !~ m/^\d+$/)) {
1534         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1535         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1536         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1537         verify_media_type($opt, $vtype, $drive->{media});
1538         $drive->{file} = $volid;
1539     }
1540
1541     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1542 }
1543
1544 sub parse_hotplug_features {
1545     my ($data) = @_;
1546
1547     my $res = {};
1548
1549     return $res if $data eq '0';
1550
1551     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1552
1553     foreach my $feature (PVE::Tools::split_list($data)) {
1554         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1555             $res->{$1} = 1;
1556         } else {
1557             die "invalid hotplug feature '$feature'\n";
1558         }
1559     }
1560     return $res;
1561 }
1562
1563 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1564 sub pve_verify_hotplug_features {
1565     my ($value, $noerr) = @_;
1566
1567     return $value if parse_hotplug_features($value);
1568
1569     return undef if $noerr;
1570
1571     die "unable to parse hotplug option\n";
1572 }
1573
1574 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1575 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1576 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1577 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1578 #        [,iothread=on][,serial=serial][,model=model]
1579
1580 sub parse_drive {
1581     my ($key, $data) = @_;
1582
1583     my ($interface, $index);
1584
1585     if ($key =~ m/^([^\d]+)(\d+)$/) {
1586         $interface = $1;
1587         $index = $2;
1588     } else {
1589         return undef;
1590     }
1591
1592     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1593                                      : $confdesc->{$key}->{format};
1594     if (!$desc) {
1595         warn "invalid drive key: $key\n";
1596         return undef;
1597     }
1598     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1599     return undef if !$res;
1600     $res->{interface} = $interface;
1601     $res->{index} = $index;
1602
1603     my $error = 0;
1604     foreach my $opt (qw(bps bps_rd bps_wr)) {
1605         if (my $bps = defined(delete $res->{$opt})) {
1606             if (defined($res->{"m$opt"})) {
1607                 warn "both $opt and m$opt specified\n";
1608                 ++$error;
1609                 next;
1610             }
1611             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1612         }
1613     }
1614
1615     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1616     for my $requirement (
1617         [mbps_max => 'mbps'],
1618         [mbps_rd_max => 'mbps_rd'],
1619         [mbps_wr_max => 'mbps_wr'],
1620         [miops_max => 'miops'],
1621         [miops_rd_max => 'miops_rd'],
1622         [miops_wr_max => 'miops_wr'],
1623         [bps_max_length => 'mbps_max'],
1624         [bps_rd_max_length => 'mbps_rd_max'],
1625         [bps_wr_max_length => 'mbps_wr_max'],
1626         [iops_max_length => 'iops_max'],
1627         [iops_rd_max_length => 'iops_rd_max'],
1628         [iops_wr_max_length => 'iops_wr_max']) {
1629         my ($option, $requires) = @$requirement;
1630         if ($res->{$option} && !$res->{$requires}) {
1631             warn "$option requires $requires\n";
1632             ++$error;
1633         }
1634     }
1635
1636     return undef if $error;
1637
1638     return undef if $res->{mbps_rd} && $res->{mbps};
1639     return undef if $res->{mbps_wr} && $res->{mbps};
1640     return undef if $res->{iops_rd} && $res->{iops};
1641     return undef if $res->{iops_wr} && $res->{iops};
1642
1643     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1644         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1645         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1646         return undef if $res->{interface} eq 'virtio';
1647     }
1648
1649     if (my $size = $res->{size}) {
1650         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1651     }
1652
1653     return $res;
1654 }
1655
1656 sub print_drive {
1657     my ($drive) = @_;
1658     my $data = { %$drive };
1659     delete $data->{$_} for qw(index interface);
1660     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1661 }
1662
1663 sub scsi_inquiry {
1664     my($fh, $noerr) = @_;
1665
1666     my $SG_IO = 0x2285;
1667     my $SG_GET_VERSION_NUM = 0x2282;
1668
1669     my $versionbuf = "\x00" x 8;
1670     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1671     if (!$ret) {
1672         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1673         return undef;
1674     }
1675     my $version = unpack("I", $versionbuf);
1676     if ($version < 30000) {
1677         die "scsi generic interface too old\n"  if !$noerr;
1678         return undef;
1679     }
1680
1681     my $buf = "\x00" x 36;
1682     my $sensebuf = "\x00" x 8;
1683     my $cmd = pack("C x3 C x1", 0x12, 36);
1684
1685     # see /usr/include/scsi/sg.h
1686     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1687
1688     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1689                       length($sensebuf), 0, length($buf), $buf,
1690                       $cmd, $sensebuf, 6000);
1691
1692     $ret = ioctl($fh, $SG_IO, $packet);
1693     if (!$ret) {
1694         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1695         return undef;
1696     }
1697
1698     my @res = unpack($sg_io_hdr_t, $packet);
1699     if ($res[17] || $res[18]) {
1700         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1701         return undef;
1702     }
1703
1704     my $res = {};
1705     (my $byte0, my $byte1, $res->{vendor},
1706      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1707
1708     $res->{removable} = $byte1 & 128 ? 1 : 0;
1709     $res->{type} = $byte0 & 31;
1710
1711     return $res;
1712 }
1713
1714 sub path_is_scsi {
1715     my ($path) = @_;
1716
1717     my $fh = IO::File->new("+<$path") || return undef;
1718     my $res = scsi_inquiry($fh, 1);
1719     close($fh);
1720
1721     return $res;
1722 }
1723
1724 sub print_tabletdevice_full {
1725     my ($conf, $arch) = @_;
1726
1727     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1728
1729     # we use uhci for old VMs because tablet driver was buggy in older qemu
1730     my $usbbus;
1731     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1732         $usbbus = 'ehci';
1733     } else {
1734         $usbbus = 'uhci';
1735     }
1736
1737     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1738 }
1739
1740 sub print_keyboarddevice_full {
1741     my ($conf, $arch, $machine) = @_;
1742
1743     return undef if $arch ne 'aarch64';
1744
1745     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1746 }
1747
1748 sub print_drivedevice_full {
1749     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1750
1751     my $device = '';
1752     my $maxdev = 0;
1753
1754     if ($drive->{interface} eq 'virtio') {
1755         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1756         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1757         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1758     } elsif ($drive->{interface} eq 'scsi') {
1759
1760         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1761         my $unit = $drive->{index} % $maxdev;
1762         my $devicetype = 'hd';
1763         my $path = '';
1764         if (drive_is_cdrom($drive)) {
1765             $devicetype = 'cd';
1766         } else {
1767             if ($drive->{file} =~ m|^/|) {
1768                 $path = $drive->{file};
1769                 if (my $info = path_is_scsi($path)) {
1770                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1771                         $devicetype = 'block';
1772                     } elsif ($info->{type} == 1) { # tape
1773                         $devicetype = 'generic';
1774                     }
1775                 }
1776             } else {
1777                  $path = PVE::Storage::path($storecfg, $drive->{file});
1778             }
1779
1780             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1781             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1782             if ($path =~ m/^iscsi\:\/\// &&
1783                !min_version($version, 4, 1)) {
1784                 $devicetype = 'generic';
1785             }
1786         }
1787
1788         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1789             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1790         } else {
1791             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1792         }
1793
1794         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1795             $device .= ",rotation_rate=1";
1796         }
1797         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1798
1799     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1800         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1801         my $controller = int($drive->{index} / $maxdev);
1802         my $unit = $drive->{index} % $maxdev;
1803         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1804
1805         $device = "ide-$devicetype";
1806         if ($drive->{interface} eq 'ide') {
1807             $device .= ",bus=ide.$controller,unit=$unit";
1808         } else {
1809             $device .= ",bus=ahci$controller.$unit";
1810         }
1811         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1812
1813         if ($devicetype eq 'hd') {
1814             if (my $model = $drive->{model}) {
1815                 $model = URI::Escape::uri_unescape($model);
1816                 $device .= ",model=$model";
1817             }
1818             if ($drive->{ssd}) {
1819                 $device .= ",rotation_rate=1";
1820             }
1821         }
1822         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1823     } elsif ($drive->{interface} eq 'usb') {
1824         die "implement me";
1825         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1826     } else {
1827         die "unsupported interface type";
1828     }
1829
1830     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1831
1832     if (my $serial = $drive->{serial}) {
1833         $serial = URI::Escape::uri_unescape($serial);
1834         $device .= ",serial=$serial";
1835     }
1836
1837
1838     return $device;
1839 }
1840
1841 sub get_initiator_name {
1842     my $initiator;
1843
1844     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1845     while (defined(my $line = <$fh>)) {
1846         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1847         $initiator = $1;
1848         last;
1849     }
1850     $fh->close();
1851
1852     return $initiator;
1853 }
1854
1855 sub print_drive_full {
1856     my ($storecfg, $vmid, $drive) = @_;
1857
1858     my $path;
1859     my $volid = $drive->{file};
1860     my $format;
1861
1862     if (drive_is_cdrom($drive)) {
1863         $path = get_iso_path($storecfg, $vmid, $volid);
1864     } else {
1865         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1866         if ($storeid) {
1867             $path = PVE::Storage::path($storecfg, $volid);
1868             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1869             $format = qemu_img_format($scfg, $volname);
1870         } else {
1871             $path = $volid;
1872             $format = "raw";
1873         }
1874    }
1875
1876     my $opts = '';
1877     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1878     foreach my $o (@qemu_drive_options) {
1879         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1880     }
1881
1882     # snapshot only accepts on|off
1883     if (defined($drive->{snapshot})) {
1884         my $v = $drive->{snapshot} ? 'on' : 'off';
1885         $opts .= ",snapshot=$v";
1886     }
1887
1888     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1889         my ($dir, $qmpname) = @$type;
1890         if (my $v = $drive->{"mbps$dir"}) {
1891             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1892         }
1893         if (my $v = $drive->{"mbps${dir}_max"}) {
1894             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1895         }
1896         if (my $v = $drive->{"bps${dir}_max_length"}) {
1897             $opts .= ",throttling.bps$qmpname-max-length=$v";
1898         }
1899         if (my $v = $drive->{"iops${dir}"}) {
1900             $opts .= ",throttling.iops$qmpname=$v";
1901         }
1902         if (my $v = $drive->{"iops${dir}_max"}) {
1903             $opts .= ",throttling.iops$qmpname-max=$v";
1904         }
1905         if (my $v = $drive->{"iops${dir}_max_length"}) {
1906             $opts .= ",throttling.iops$qmpname-max-length=$v";
1907         }
1908     }
1909
1910     $opts .= ",format=$format" if $format && !$drive->{format};
1911
1912     my $cache_direct = 0;
1913
1914     if (my $cache = $drive->{cache}) {
1915         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1916     } elsif (!drive_is_cdrom($drive)) {
1917         $opts .= ",cache=none";
1918         $cache_direct = 1;
1919     }
1920
1921     # aio native works only with O_DIRECT
1922     if (!$drive->{aio}) {
1923         if($cache_direct) {
1924             $opts .= ",aio=native";
1925         } else {
1926             $opts .= ",aio=threads";
1927         }
1928     }
1929
1930     if (!drive_is_cdrom($drive)) {
1931         my $detectzeroes;
1932         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1933             $detectzeroes = 'off';
1934         } elsif ($drive->{discard}) {
1935             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1936         } else {
1937             # This used to be our default with discard not being specified:
1938             $detectzeroes = 'on';
1939         }
1940         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1941     }
1942
1943     my $pathinfo = $path ? "file=$path," : '';
1944
1945     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1946 }
1947
1948 sub print_netdevice_full {
1949     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1950
1951     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1952
1953     my $device = $net->{model};
1954     if ($net->{model} eq 'virtio') {
1955          $device = 'virtio-net-pci';
1956      };
1957
1958     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1959     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1960     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1961         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1962         my $vectors = $net->{queues} * 2 + 2;
1963         $tmpstr .= ",vectors=$vectors,mq=on";
1964     }
1965     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1966
1967     if ($use_old_bios_files) {
1968         my $romfile;
1969         if ($device eq 'virtio-net-pci') {
1970             $romfile = 'pxe-virtio.rom';
1971         } elsif ($device eq 'e1000') {
1972             $romfile = 'pxe-e1000.rom';
1973         } elsif ($device eq 'ne2k') {
1974             $romfile = 'pxe-ne2k_pci.rom';
1975         } elsif ($device eq 'pcnet') {
1976             $romfile = 'pxe-pcnet.rom';
1977         } elsif ($device eq 'rtl8139') {
1978             $romfile = 'pxe-rtl8139.rom';
1979         }
1980         $tmpstr .= ",romfile=$romfile" if $romfile;
1981     }
1982
1983     return $tmpstr;
1984 }
1985
1986 sub print_netdev_full {
1987     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1988
1989     my $i = '';
1990     if ($netid =~ m/^net(\d+)$/) {
1991         $i = int($1);
1992     }
1993
1994     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1995
1996     my $ifname = "tap${vmid}i$i";
1997
1998     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1999     die "interface name '$ifname' is too long (max 15 character)\n"
2000         if length($ifname) >= 16;
2001
2002     my $vhostparam = '';
2003     if (is_native($arch)) {
2004         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2005     }
2006
2007     my $vmname = $conf->{name} || "vm$vmid";
2008
2009     my $netdev = "";
2010     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2011
2012     if ($net->{bridge}) {
2013         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2014     } else {
2015         $netdev = "type=user,id=$netid,hostname=$vmname";
2016     }
2017
2018     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2019
2020     return $netdev;
2021 }
2022
2023 my $vga_map = {
2024     'cirrus' => 'cirrus-vga',
2025     'std' => 'VGA',
2026     'vmware' => 'vmware-svga',
2027     'virtio' => 'virtio-vga',
2028 };
2029
2030 sub print_vga_device {
2031     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2032
2033     my $type = $vga_map->{$vga->{type}};
2034     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2035         $type = 'virtio-gpu';
2036     }
2037     my $vgamem_mb = $vga->{memory};
2038
2039     my $max_outputs = '';
2040     if ($qxlnum) {
2041         $type = $id ? 'qxl' : 'qxl-vga';
2042
2043         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2044             # set max outputs so linux can have up to 4 qxl displays with one device
2045             if (min_version($machine_version, 4, 1)) {
2046                 $max_outputs = ",max_outputs=4";
2047             }
2048         }
2049     }
2050
2051     die "no devicetype for $vga->{type}\n" if !$type;
2052
2053     my $memory = "";
2054     if ($vgamem_mb) {
2055         if ($vga->{type} eq 'virtio') {
2056             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2057             $memory = ",max_hostmem=$bytes";
2058         } elsif ($qxlnum) {
2059             # from https://www.spice-space.org/multiple-monitors.html
2060             $memory = ",vgamem_mb=$vga->{memory}";
2061             my $ram = $vgamem_mb * 4;
2062             my $vram = $vgamem_mb * 2;
2063             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2064         } else {
2065             $memory = ",vgamem_mb=$vga->{memory}";
2066         }
2067     } elsif ($qxlnum && $id) {
2068         $memory = ",ram_size=67108864,vram_size=33554432";
2069     }
2070
2071     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2072     my $vgaid = "vga" . ($id // '');
2073     my $pciaddr;
2074
2075     if ($q35 && $vgaid eq 'vga') {
2076         # the first display uses pcie.0 bus on q35 machines
2077         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2078     } else {
2079         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2080     }
2081
2082     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2083 }
2084
2085 sub drive_is_cloudinit {
2086     my ($drive) = @_;
2087     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2088 }
2089
2090 sub drive_is_cdrom {
2091     my ($drive, $exclude_cloudinit) = @_;
2092
2093     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2094
2095     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2096
2097 }
2098
2099 sub parse_number_sets {
2100     my ($set) = @_;
2101     my $res = [];
2102     foreach my $part (split(/;/, $set)) {
2103         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2104             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2105             push @$res, [ $1, $2 ];
2106         } else {
2107             die "invalid range: $part\n";
2108         }
2109     }
2110     return $res;
2111 }
2112
2113 sub parse_numa {
2114     my ($data) = @_;
2115
2116     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2117     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2118     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2119     return $res;
2120 }
2121
2122 sub parse_hostpci {
2123     my ($value) = @_;
2124
2125     return undef if !$value;
2126
2127     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2128
2129     my @idlist = split(/;/, $res->{host});
2130     delete $res->{host};
2131     foreach my $id (@idlist) {
2132         my $devs = PVE::SysFSTools::lspci($id);
2133         die "no PCI device found for '$id'\n" if !scalar(@$devs);
2134         push @{$res->{pciid}}, @$devs;
2135     }
2136     return $res;
2137 }
2138
2139 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2140 sub parse_net {
2141     my ($data) = @_;
2142
2143     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2144     if ($@) {
2145         warn $@;
2146         return undef;
2147     }
2148     if (!defined($res->{macaddr})) {
2149         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2150         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2151     }
2152     return $res;
2153 }
2154
2155 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2156 sub parse_ipconfig {
2157     my ($data) = @_;
2158
2159     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2160     if ($@) {
2161         warn $@;
2162         return undef;
2163     }
2164
2165     if ($res->{gw} && !$res->{ip}) {
2166         warn 'gateway specified without specifying an IP address';
2167         return undef;
2168     }
2169     if ($res->{gw6} && !$res->{ip6}) {
2170         warn 'IPv6 gateway specified without specifying an IPv6 address';
2171         return undef;
2172     }
2173     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2174         warn 'gateway specified together with DHCP';
2175         return undef;
2176     }
2177     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2178         # gw6 + auto/dhcp
2179         warn "IPv6 gateway specified together with $res->{ip6} address";
2180         return undef;
2181     }
2182
2183     if (!$res->{ip} && !$res->{ip6}) {
2184         return { ip => 'dhcp', ip6 => 'dhcp' };
2185     }
2186
2187     return $res;
2188 }
2189
2190 sub print_net {
2191     my $net = shift;
2192
2193     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2194 }
2195
2196 sub add_random_macs {
2197     my ($settings) = @_;
2198
2199     foreach my $opt (keys %$settings) {
2200         next if $opt !~ m/^net(\d+)$/;
2201         my $net = parse_net($settings->{$opt});
2202         next if !$net;
2203         $settings->{$opt} = print_net($net);
2204     }
2205 }
2206
2207 sub vm_is_volid_owner {
2208     my ($storecfg, $vmid, $volid) = @_;
2209
2210     if ($volid !~  m|^/|) {
2211         my ($path, $owner);
2212         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2213         if ($owner && ($owner == $vmid)) {
2214             return 1;
2215         }
2216     }
2217
2218     return undef;
2219 }
2220
2221 sub vmconfig_register_unused_drive {
2222     my ($storecfg, $vmid, $conf, $drive) = @_;
2223
2224     if (drive_is_cloudinit($drive)) {
2225         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2226         warn $@ if $@;
2227     } elsif (!drive_is_cdrom($drive)) {
2228         my $volid = $drive->{file};
2229         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2230             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2231         }
2232     }
2233 }
2234
2235 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2236 my $smbios1_fmt = {
2237     uuid => {
2238         type => 'string',
2239         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2240         format_description => 'UUID',
2241         description => "Set SMBIOS1 UUID.",
2242         optional => 1,
2243     },
2244     version => {
2245         type => 'string',
2246         pattern => '[A-Za-z0-9+\/]+={0,2}',
2247         format_description => 'Base64 encoded string',
2248         description => "Set SMBIOS1 version.",
2249         optional => 1,
2250     },
2251     serial => {
2252         type => 'string',
2253         pattern => '[A-Za-z0-9+\/]+={0,2}',
2254         format_description => 'Base64 encoded string',
2255         description => "Set SMBIOS1 serial number.",
2256         optional => 1,
2257     },
2258     manufacturer => {
2259         type => 'string',
2260         pattern => '[A-Za-z0-9+\/]+={0,2}',
2261         format_description => 'Base64 encoded string',
2262         description => "Set SMBIOS1 manufacturer.",
2263         optional => 1,
2264     },
2265     product => {
2266         type => 'string',
2267         pattern => '[A-Za-z0-9+\/]+={0,2}',
2268         format_description => 'Base64 encoded string',
2269         description => "Set SMBIOS1 product ID.",
2270         optional => 1,
2271     },
2272     sku => {
2273         type => 'string',
2274         pattern => '[A-Za-z0-9+\/]+={0,2}',
2275         format_description => 'Base64 encoded string',
2276         description => "Set SMBIOS1 SKU string.",
2277         optional => 1,
2278     },
2279     family => {
2280         type => 'string',
2281         pattern => '[A-Za-z0-9+\/]+={0,2}',
2282         format_description => 'Base64 encoded string',
2283         description => "Set SMBIOS1 family string.",
2284         optional => 1,
2285     },
2286     base64 => {
2287         type => 'boolean',
2288         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2289         optional => 1,
2290     },
2291 };
2292
2293 sub parse_smbios1 {
2294     my ($data) = @_;
2295
2296     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2297     warn $@ if $@;
2298     return $res;
2299 }
2300
2301 sub print_smbios1 {
2302     my ($smbios1) = @_;
2303     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2304 }
2305
2306 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2307
2308 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2309 sub verify_bootdisk {
2310     my ($value, $noerr) = @_;
2311
2312     return $value if is_valid_drivename($value);
2313
2314     return undef if $noerr;
2315
2316     die "invalid boot disk '$value'\n";
2317 }
2318
2319 sub parse_watchdog {
2320     my ($value) = @_;
2321
2322     return undef if !$value;
2323
2324     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2325     warn $@ if $@;
2326     return $res;
2327 }
2328
2329 sub parse_guest_agent {
2330     my ($value) = @_;
2331
2332     return {} if !defined($value->{agent});
2333
2334     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2335     warn $@ if $@;
2336
2337     # if the agent is disabled ignore the other potentially set properties
2338     return {} if !$res->{enabled};
2339     return $res;
2340 }
2341
2342 sub parse_vga {
2343     my ($value) = @_;
2344
2345     return {} if !$value;
2346     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2347     warn $@ if $@;
2348     return $res;
2349 }
2350
2351 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2352 sub verify_usb_device {
2353     my ($value, $noerr) = @_;
2354
2355     return $value if parse_usb_device($value);
2356
2357     return undef if $noerr;
2358
2359     die "unable to parse usb device\n";
2360 }
2361
2362 # add JSON properties for create and set function
2363 sub json_config_properties {
2364     my $prop = shift;
2365
2366     foreach my $opt (keys %$confdesc) {
2367         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2368         $prop->{$opt} = $confdesc->{$opt};
2369     }
2370
2371     return $prop;
2372 }
2373
2374 # return copy of $confdesc_cloudinit to generate documentation
2375 sub cloudinit_config_properties {
2376
2377     return dclone($confdesc_cloudinit);
2378 }
2379
2380 sub check_type {
2381     my ($key, $value) = @_;
2382
2383     die "unknown setting '$key'\n" if !$confdesc->{$key};
2384
2385     my $type = $confdesc->{$key}->{type};
2386
2387     if (!defined($value)) {
2388         die "got undefined value\n";
2389     }
2390
2391     if ($value =~ m/[\n\r]/) {
2392         die "property contains a line feed\n";
2393     }
2394
2395     if ($type eq 'boolean') {
2396         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2397         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2398         die "type check ('boolean') failed - got '$value'\n";
2399     } elsif ($type eq 'integer') {
2400         return int($1) if $value =~ m/^(\d+)$/;
2401         die "type check ('integer') failed - got '$value'\n";
2402     } elsif ($type eq 'number') {
2403         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2404         die "type check ('number') failed - got '$value'\n";
2405     } elsif ($type eq 'string') {
2406         if (my $fmt = $confdesc->{$key}->{format}) {
2407             PVE::JSONSchema::check_format($fmt, $value);
2408             return $value;
2409         }
2410         $value =~ s/^\"(.*)\"$/$1/;
2411         return $value;
2412     } else {
2413         die "internal error"
2414     }
2415 }
2416
2417 sub destroy_vm {
2418     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2419
2420     my $conf = PVE::QemuConfig->load_config($vmid);
2421
2422     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2423
2424     if ($conf->{template}) {
2425         # check if any base image is still used by a linked clone
2426         foreach_drive($conf, sub {
2427                 my ($ds, $drive) = @_;
2428                 return if drive_is_cdrom($drive);
2429
2430                 my $volid = $drive->{file};
2431                 return if !$volid || $volid =~ m|^/|;
2432
2433                 die "base volume '$volid' is still in use by linked cloned\n"
2434                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2435
2436         });
2437     }
2438
2439     # only remove disks owned by this VM
2440     foreach_drive($conf, sub {
2441         my ($ds, $drive) = @_;
2442         return if drive_is_cdrom($drive, 1);
2443
2444         my $volid = $drive->{file};
2445         return if !$volid || $volid =~ m|^/|;
2446
2447         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2448         return if !$path || !$owner || ($owner != $vmid);
2449
2450         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2451         warn "Could not remove disk '$volid', check manually: $@" if $@;
2452     });
2453
2454     # also remove unused disk
2455     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2456     PVE::Storage::foreach_volid($vmdisks, sub {
2457         my ($volid, $sid, $volname, $d) = @_;
2458         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2459         warn $@ if $@;
2460     });
2461
2462     if (defined $replacement_conf) {
2463         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2464     } else {
2465         PVE::QemuConfig->destroy_config($vmid);
2466     }
2467 }
2468
2469 sub parse_vm_config {
2470     my ($filename, $raw) = @_;
2471
2472     return undef if !defined($raw);
2473
2474     my $res = {
2475         digest => Digest::SHA::sha1_hex($raw),
2476         snapshots => {},
2477         pending => {},
2478     };
2479
2480     $filename =~ m|/qemu-server/(\d+)\.conf$|
2481         || die "got strange filename '$filename'";
2482
2483     my $vmid = $1;
2484
2485     my $conf = $res;
2486     my $descr;
2487     my $section = '';
2488
2489     my @lines = split(/\n/, $raw);
2490     foreach my $line (@lines) {
2491         next if $line =~ m/^\s*$/;
2492
2493         if ($line =~ m/^\[PENDING\]\s*$/i) {
2494             $section = 'pending';
2495             if (defined($descr)) {
2496                 $descr =~ s/\s+$//;
2497                 $conf->{description} = $descr;
2498             }
2499             $descr = undef;
2500             $conf = $res->{$section} = {};
2501             next;
2502
2503         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2504             $section = $1;
2505             if (defined($descr)) {
2506                 $descr =~ s/\s+$//;
2507                 $conf->{description} = $descr;
2508             }
2509             $descr = undef;
2510             $conf = $res->{snapshots}->{$section} = {};
2511             next;
2512         }
2513
2514         if ($line =~ m/^\#(.*)\s*$/) {
2515             $descr = '' if !defined($descr);
2516             $descr .= PVE::Tools::decode_text($1) . "\n";
2517             next;
2518         }
2519
2520         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2521             $descr = '' if !defined($descr);
2522             $descr .= PVE::Tools::decode_text($2);
2523         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2524             $conf->{snapstate} = $1;
2525         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2526             my $key = $1;
2527             my $value = $2;
2528             $conf->{$key} = $value;
2529         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2530             my $value = $1;
2531             if ($section eq 'pending') {
2532                 $conf->{delete} = $value; # we parse this later
2533             } else {
2534                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2535             }
2536         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2537             my $key = $1;
2538             my $value = $2;
2539             eval { $value = check_type($key, $value); };
2540             if ($@) {
2541                 warn "vm $vmid - unable to parse value of '$key' - $@";
2542             } else {
2543                 $key = 'ide2' if $key eq 'cdrom';
2544                 my $fmt = $confdesc->{$key}->{format};
2545                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2546                     my $v = parse_drive($key, $value);
2547                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2548                         $v->{file} = $volid;
2549                         $value = print_drive($v);
2550                     } else {
2551                         warn "vm $vmid - unable to parse value of '$key'\n";
2552                         next;
2553                     }
2554                 }
2555
2556                 $conf->{$key} = $value;
2557             }
2558         }
2559     }
2560
2561     if (defined($descr)) {
2562         $descr =~ s/\s+$//;
2563         $conf->{description} = $descr;
2564     }
2565     delete $res->{snapstate}; # just to be sure
2566
2567     return $res;
2568 }
2569
2570 sub write_vm_config {
2571     my ($filename, $conf) = @_;
2572
2573     delete $conf->{snapstate}; # just to be sure
2574
2575     if ($conf->{cdrom}) {
2576         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2577         $conf->{ide2} = $conf->{cdrom};
2578         delete $conf->{cdrom};
2579     }
2580
2581     # we do not use 'smp' any longer
2582     if ($conf->{sockets}) {
2583         delete $conf->{smp};
2584     } elsif ($conf->{smp}) {
2585         $conf->{sockets} = $conf->{smp};
2586         delete $conf->{cores};
2587         delete $conf->{smp};
2588     }
2589
2590     my $used_volids = {};
2591
2592     my $cleanup_config = sub {
2593         my ($cref, $pending, $snapname) = @_;
2594
2595         foreach my $key (keys %$cref) {
2596             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2597                 $key eq 'snapstate' || $key eq 'pending';
2598             my $value = $cref->{$key};
2599             if ($key eq 'delete') {
2600                 die "propertry 'delete' is only allowed in [PENDING]\n"
2601                     if !$pending;
2602                 # fixme: check syntax?
2603                 next;
2604             }
2605             eval { $value = check_type($key, $value); };
2606             die "unable to parse value of '$key' - $@" if $@;
2607
2608             $cref->{$key} = $value;
2609
2610             if (!$snapname && is_valid_drivename($key)) {
2611                 my $drive = parse_drive($key, $value);
2612                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2613             }
2614         }
2615     };
2616
2617     &$cleanup_config($conf);
2618
2619     &$cleanup_config($conf->{pending}, 1);
2620
2621     foreach my $snapname (keys %{$conf->{snapshots}}) {
2622         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2623         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2624     }
2625
2626     # remove 'unusedX' settings if we re-add a volume
2627     foreach my $key (keys %$conf) {
2628         my $value = $conf->{$key};
2629         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2630             delete $conf->{$key};
2631         }
2632     }
2633
2634     my $generate_raw_config = sub {
2635         my ($conf, $pending) = @_;
2636
2637         my $raw = '';
2638
2639         # add description as comment to top of file
2640         if (defined(my $descr = $conf->{description})) {
2641             if ($descr) {
2642                 foreach my $cl (split(/\n/, $descr)) {
2643                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2644                 }
2645             } else {
2646                 $raw .= "#\n" if $pending;
2647             }
2648         }
2649
2650         foreach my $key (sort keys %$conf) {
2651             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2652             $raw .= "$key: $conf->{$key}\n";
2653         }
2654         return $raw;
2655     };
2656
2657     my $raw = &$generate_raw_config($conf);
2658
2659     if (scalar(keys %{$conf->{pending}})){
2660         $raw .= "\n[PENDING]\n";
2661         $raw .= &$generate_raw_config($conf->{pending}, 1);
2662     }
2663
2664     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2665         $raw .= "\n[$snapname]\n";
2666         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2667     }
2668
2669     return $raw;
2670 }
2671
2672 sub load_defaults {
2673
2674     my $res = {};
2675
2676     # we use static defaults from our JSON schema configuration
2677     foreach my $key (keys %$confdesc) {
2678         if (defined(my $default = $confdesc->{$key}->{default})) {
2679             $res->{$key} = $default;
2680         }
2681     }
2682
2683     return $res;
2684 }
2685
2686 sub config_list {
2687     my $vmlist = PVE::Cluster::get_vmlist();
2688     my $res = {};
2689     return $res if !$vmlist || !$vmlist->{ids};
2690     my $ids = $vmlist->{ids};
2691     my $nodename = nodename();
2692
2693     foreach my $vmid (keys %$ids) {
2694         my $d = $ids->{$vmid};
2695         next if !$d->{node} || $d->{node} ne $nodename;
2696         next if !$d->{type} || $d->{type} ne 'qemu';
2697         $res->{$vmid}->{exists} = 1;
2698     }
2699     return $res;
2700 }
2701
2702 # test if VM uses local resources (to prevent migration)
2703 sub check_local_resources {
2704     my ($conf, $noerr) = @_;
2705
2706     my @loc_res = ();
2707
2708     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2709     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2710
2711     push @loc_res, "ivshmem" if $conf->{ivshmem};
2712
2713     foreach my $k (keys %$conf) {
2714         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2715         # sockets are safe: they will recreated be on the target side post-migrate
2716         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2717         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2718     }
2719
2720     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2721
2722     return \@loc_res;
2723 }
2724
2725 # check if used storages are available on all nodes (use by migrate)
2726 sub check_storage_availability {
2727     my ($storecfg, $conf, $node) = @_;
2728
2729     foreach_drive($conf, sub {
2730         my ($ds, $drive) = @_;
2731
2732         my $volid = $drive->{file};
2733         return if !$volid;
2734
2735         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2736         return if !$sid;
2737
2738         # check if storage is available on both nodes
2739         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2740         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2741    });
2742 }
2743
2744 # list nodes where all VM images are available (used by has_feature API)
2745 sub shared_nodes {
2746     my ($conf, $storecfg) = @_;
2747
2748     my $nodelist = PVE::Cluster::get_nodelist();
2749     my $nodehash = { map { $_ => 1 } @$nodelist };
2750     my $nodename = nodename();
2751
2752     foreach_drive($conf, sub {
2753         my ($ds, $drive) = @_;
2754
2755         my $volid = $drive->{file};
2756         return if !$volid;
2757
2758         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2759         if ($storeid) {
2760             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2761             if ($scfg->{disable}) {
2762                 $nodehash = {};
2763             } elsif (my $avail = $scfg->{nodes}) {
2764                 foreach my $node (keys %$nodehash) {
2765                     delete $nodehash->{$node} if !$avail->{$node};
2766                 }
2767             } elsif (!$scfg->{shared}) {
2768                 foreach my $node (keys %$nodehash) {
2769                     delete $nodehash->{$node} if $node ne $nodename
2770                 }
2771             }
2772         }
2773     });
2774
2775     return $nodehash
2776 }
2777
2778 sub check_local_storage_availability {
2779     my ($conf, $storecfg) = @_;
2780
2781     my $nodelist = PVE::Cluster::get_nodelist();
2782     my $nodehash = { map { $_ => {} } @$nodelist };
2783
2784     foreach_drive($conf, sub {
2785         my ($ds, $drive) = @_;
2786
2787         my $volid = $drive->{file};
2788         return if !$volid;
2789
2790         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2791         if ($storeid) {
2792             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2793
2794             if ($scfg->{disable}) {
2795                 foreach my $node (keys %$nodehash) {
2796                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2797                 }
2798             } elsif (my $avail = $scfg->{nodes}) {
2799                 foreach my $node (keys %$nodehash) {
2800                     if (!$avail->{$node}) {
2801                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2802                     }
2803                 }
2804             }
2805         }
2806     });
2807
2808     foreach my $node (values %$nodehash) {
2809         if (my $unavail = $node->{unavailable_storages}) {
2810             $node->{unavailable_storages} = [ sort keys %$unavail ];
2811         }
2812     }
2813
2814     return $nodehash
2815 }
2816
2817 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2818 sub check_running {
2819     my ($vmid, $nocheck, $node) = @_;
2820
2821     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2822     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2823 }
2824
2825 sub vzlist {
2826
2827     my $vzlist = config_list();
2828
2829     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2830
2831     while (defined(my $de = $fd->read)) {
2832         next if $de !~ m/^(\d+)\.pid$/;
2833         my $vmid = $1;
2834         next if !defined($vzlist->{$vmid});
2835         if (my $pid = check_running($vmid)) {
2836             $vzlist->{$vmid}->{pid} = $pid;
2837         }
2838     }
2839
2840     return $vzlist;
2841 }
2842
2843 sub disksize {
2844     my ($storecfg, $conf) = @_;
2845
2846     my $bootdisk = $conf->{bootdisk};
2847     return undef if !$bootdisk;
2848     return undef if !is_valid_drivename($bootdisk);
2849
2850     return undef if !$conf->{$bootdisk};
2851
2852     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2853     return undef if !defined($drive);
2854
2855     return undef if drive_is_cdrom($drive);
2856
2857     my $volid = $drive->{file};
2858     return undef if !$volid;
2859
2860     return $drive->{size};
2861 }
2862
2863 our $vmstatus_return_properties = {
2864     vmid => get_standard_option('pve-vmid'),
2865     status => {
2866         description => "Qemu process status.",
2867         type => 'string',
2868         enum => ['stopped', 'running'],
2869     },
2870     maxmem => {
2871         description => "Maximum memory in bytes.",
2872         type => 'integer',
2873         optional => 1,
2874         renderer => 'bytes',
2875     },
2876     maxdisk => {
2877         description => "Root disk size in bytes.",
2878         type => 'integer',
2879         optional => 1,
2880         renderer => 'bytes',
2881     },
2882     name => {
2883         description => "VM name.",
2884         type => 'string',
2885         optional => 1,
2886     },
2887     qmpstatus => {
2888         description => "Qemu QMP agent status.",
2889         type => 'string',
2890         optional => 1,
2891     },
2892     pid => {
2893         description => "PID of running qemu process.",
2894         type => 'integer',
2895         optional => 1,
2896     },
2897     uptime => {
2898         description => "Uptime.",
2899         type => 'integer',
2900         optional => 1,
2901         renderer => 'duration',
2902     },
2903     cpus => {
2904         description => "Maximum usable CPUs.",
2905         type => 'number',
2906         optional => 1,
2907     },
2908     lock => {
2909         description => "The current config lock, if any.",
2910         type => 'string',
2911         optional => 1,
2912     },
2913     tags => {
2914         description  => "The current configured tags, if any",
2915         type => 'string',
2916         optional => 1,
2917     },
2918 };
2919
2920 my $last_proc_pid_stat;
2921
2922 # get VM status information
2923 # This must be fast and should not block ($full == false)
2924 # We only query KVM using QMP if $full == true (this can be slow)
2925 sub vmstatus {
2926     my ($opt_vmid, $full) = @_;
2927
2928     my $res = {};
2929
2930     my $storecfg = PVE::Storage::config();
2931
2932     my $list = vzlist();
2933     my $defaults = load_defaults();
2934
2935     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2936
2937     my $cpucount = $cpuinfo->{cpus} || 1;
2938
2939     foreach my $vmid (keys %$list) {
2940         next if $opt_vmid && ($vmid ne $opt_vmid);
2941
2942         my $conf = PVE::QemuConfig->load_config($vmid);
2943
2944         my $d = { vmid => $vmid };
2945         $d->{pid} = $list->{$vmid}->{pid};
2946
2947         # fixme: better status?
2948         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2949
2950         my $size = disksize($storecfg, $conf);
2951         if (defined($size)) {
2952             $d->{disk} = 0; # no info available
2953             $d->{maxdisk} = $size;
2954         } else {
2955             $d->{disk} = 0;
2956             $d->{maxdisk} = 0;
2957         }
2958
2959         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2960             * ($conf->{cores} || $defaults->{cores});
2961         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2962         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2963
2964         $d->{name} = $conf->{name} || "VM $vmid";
2965         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2966             : $defaults->{memory}*(1024*1024);
2967
2968         if ($conf->{balloon}) {
2969             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2970             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2971                 : $defaults->{shares};
2972         }
2973
2974         $d->{uptime} = 0;
2975         $d->{cpu} = 0;
2976         $d->{mem} = 0;
2977
2978         $d->{netout} = 0;
2979         $d->{netin} = 0;
2980
2981         $d->{diskread} = 0;
2982         $d->{diskwrite} = 0;
2983
2984         $d->{template} = PVE::QemuConfig->is_template($conf);
2985
2986         $d->{serial} = 1 if conf_has_serial($conf);
2987         $d->{lock} = $conf->{lock} if $conf->{lock};
2988         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2989
2990         $res->{$vmid} = $d;
2991     }
2992
2993     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2994     foreach my $dev (keys %$netdev) {
2995         next if $dev !~ m/^tap([1-9]\d*)i/;
2996         my $vmid = $1;
2997         my $d = $res->{$vmid};
2998         next if !$d;
2999
3000         $d->{netout} += $netdev->{$dev}->{receive};
3001         $d->{netin} += $netdev->{$dev}->{transmit};
3002
3003         if ($full) {
3004             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3005             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3006         }
3007
3008     }
3009
3010     my $ctime = gettimeofday;
3011
3012     foreach my $vmid (keys %$list) {
3013
3014         my $d = $res->{$vmid};
3015         my $pid = $d->{pid};
3016         next if !$pid;
3017
3018         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3019         next if !$pstat; # not running
3020
3021         my $used = $pstat->{utime} + $pstat->{stime};
3022
3023         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3024
3025         if ($pstat->{vsize}) {
3026             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3027         }
3028
3029         my $old = $last_proc_pid_stat->{$pid};
3030         if (!$old) {
3031             $last_proc_pid_stat->{$pid} = {
3032                 time => $ctime,
3033                 used => $used,
3034                 cpu => 0,
3035             };
3036             next;
3037         }
3038
3039         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3040
3041         if ($dtime > 1000) {
3042             my $dutime = $used -  $old->{used};
3043
3044             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3045             $last_proc_pid_stat->{$pid} = {
3046                 time => $ctime,
3047                 used => $used,
3048                 cpu => $d->{cpu},
3049             };
3050         } else {
3051             $d->{cpu} = $old->{cpu};
3052         }
3053     }
3054
3055     return $res if !$full;
3056
3057     my $qmpclient = PVE::QMPClient->new();
3058
3059     my $ballooncb = sub {
3060         my ($vmid, $resp) = @_;
3061
3062         my $info = $resp->{'return'};
3063         return if !$info->{max_mem};
3064
3065         my $d = $res->{$vmid};
3066
3067         # use memory assigned to VM
3068         $d->{maxmem} = $info->{max_mem};
3069         $d->{balloon} = $info->{actual};
3070
3071         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3072             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3073             $d->{freemem} = $info->{free_mem};
3074         }
3075
3076         $d->{ballooninfo} = $info;
3077     };
3078
3079     my $blockstatscb = sub {
3080         my ($vmid, $resp) = @_;
3081         my $data = $resp->{'return'} || [];
3082         my $totalrdbytes = 0;
3083         my $totalwrbytes = 0;
3084
3085         for my $blockstat (@$data) {
3086             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3087             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3088
3089             $blockstat->{device} =~ s/drive-//;
3090             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3091         }
3092         $res->{$vmid}->{diskread} = $totalrdbytes;
3093         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3094     };
3095
3096     my $statuscb = sub {
3097         my ($vmid, $resp) = @_;
3098
3099         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3100         # this fails if ballon driver is not loaded, so this must be
3101         # the last commnand (following command are aborted if this fails).
3102         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3103
3104         my $status = 'unknown';
3105         if (!defined($status = $resp->{'return'}->{status})) {
3106             warn "unable to get VM status\n";
3107             return;
3108         }
3109
3110         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3111     };
3112
3113     foreach my $vmid (keys %$list) {
3114         next if $opt_vmid && ($vmid ne $opt_vmid);
3115         next if !$res->{$vmid}->{pid}; # not running
3116         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3117     }
3118
3119     $qmpclient->queue_execute(undef, 2);
3120
3121     foreach my $vmid (keys %$list) {
3122         next if $opt_vmid && ($vmid ne $opt_vmid);
3123         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3124     }
3125
3126     return $res;
3127 }
3128
3129 sub foreach_drive {
3130     my ($conf, $func, @param) = @_;
3131
3132     foreach my $ds (valid_drive_names()) {
3133         next if !defined($conf->{$ds});
3134
3135         my $drive = parse_drive($ds, $conf->{$ds});
3136         next if !$drive;
3137
3138         &$func($ds, $drive, @param);
3139     }
3140 }
3141
3142 sub foreach_volid {
3143     my ($conf, $func, @param) = @_;
3144
3145     my $volhash = {};
3146
3147     my $test_volid = sub {
3148         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3149
3150         return if !$volid;
3151
3152         $volhash->{$volid}->{cdrom} //= 1;
3153         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3154
3155         $volhash->{$volid}->{replicate} //= 0;
3156         $volhash->{$volid}->{replicate} = 1 if $replicate;
3157
3158         $volhash->{$volid}->{shared} //= 0;
3159         $volhash->{$volid}->{shared} = 1 if $shared;
3160
3161         $volhash->{$volid}->{referenced_in_config} //= 0;
3162         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3163
3164         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3165             if defined($snapname);
3166         $volhash->{$volid}->{size} = $size if $size;
3167     };
3168
3169     foreach_drive($conf, sub {
3170         my ($ds, $drive) = @_;
3171         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3172     });
3173
3174     foreach my $snapname (keys %{$conf->{snapshots}}) {
3175         my $snap = $conf->{snapshots}->{$snapname};
3176         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3177         foreach_drive($snap, sub {
3178             my ($ds, $drive) = @_;
3179             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3180         });
3181     }
3182
3183     foreach my $volid (keys %$volhash) {
3184         &$func($volid, $volhash->{$volid}, @param);
3185     }
3186 }
3187
3188 sub conf_has_serial {
3189     my ($conf) = @_;
3190
3191     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3192         if ($conf->{"serial$i"}) {
3193             return 1;
3194         }
3195     }
3196
3197     return 0;
3198 }
3199
3200 sub conf_has_audio {
3201     my ($conf, $id) = @_;
3202
3203     $id //= 0;
3204     my $audio = $conf->{"audio$id"};
3205     return undef if !defined($audio);
3206
3207     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3208     my $audiodriver = $audioproperties->{driver} // 'spice';
3209
3210     return {
3211         dev => $audioproperties->{device},
3212         dev_id => "audiodev$id",
3213         backend => $audiodriver,
3214         backend_id => "$audiodriver-backend${id}",
3215     };
3216 }
3217
3218 sub vga_conf_has_spice {
3219     my ($vga) = @_;
3220
3221     my $vgaconf = parse_vga($vga);
3222     my $vgatype = $vgaconf->{type};
3223     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3224
3225     return $1 || 1;
3226 }
3227
3228 sub is_native($) {
3229     my ($arch) = @_;
3230     return get_host_arch() eq $arch;
3231 }
3232
3233 sub get_vm_arch {
3234     my ($conf) = @_;
3235     return $conf->{arch} // get_host_arch();
3236 }
3237
3238 my $default_machines = {
3239     x86_64 => 'pc',
3240     aarch64 => 'virt',
3241 };
3242
3243 sub get_vm_machine {
3244     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3245
3246     my $machine = $forcemachine || $conf->{machine};
3247
3248     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3249         $arch //= 'x86_64';
3250         $machine ||= $default_machines->{$arch};
3251         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3252     }
3253
3254     return $machine;
3255 }
3256
3257 sub get_ovmf_files($) {
3258     my ($arch) = @_;
3259
3260     my $ovmf = $OVMF->{$arch}
3261         or die "no OVMF images known for architecture '$arch'\n";
3262
3263     return @$ovmf;
3264 }
3265
3266 my $Arch2Qemu = {
3267     aarch64 => '/usr/bin/qemu-system-aarch64',
3268     x86_64 => '/usr/bin/qemu-system-x86_64',
3269 };
3270 sub get_command_for_arch($) {
3271     my ($arch) = @_;
3272     return '/usr/bin/kvm' if is_native($arch);
3273
3274     my $cmd = $Arch2Qemu->{$arch}
3275         or die "don't know how to emulate architecture '$arch'\n";
3276     return $cmd;
3277 }
3278
3279 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3280 # to use in a QEMU command line (-cpu element), first array_intersect the result
3281 # of query_supported_ with query_understood_. This is necessary because:
3282 #
3283 # a) query_understood_ returns flags the host cannot use and
3284 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3285 #    flags, but CPU settings - with most of them being flags. Those settings
3286 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3287 #
3288 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3289 # expensive. If you need the value returned from this, you can get it much
3290 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3291 # $accel being 'kvm' or 'tcg'.
3292 #
3293 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3294 # changes, automatically populating pmxcfs.
3295 #
3296 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3297 # since kvm and tcg machines support different flags
3298 #
3299 sub query_supported_cpu_flags {
3300     my ($arch) = @_;
3301
3302     $arch //= get_host_arch();
3303     my $default_machine = $default_machines->{$arch};
3304
3305     my $flags = {};
3306
3307     # FIXME: Once this is merged, the code below should work for ARM as well:
3308     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3309     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3310         $arch eq "aarch64";
3311
3312     my $kvm_supported = defined(kvm_version());
3313     my $qemu_cmd = get_command_for_arch($arch);
3314     my $fakevmid = -1;
3315     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3316
3317     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3318     my $query_supported_run_qemu = sub {
3319         my ($kvm) = @_;
3320
3321         my $flags = {};
3322         my $cmd = [
3323             $qemu_cmd,
3324             '-machine', $default_machine,
3325             '-display', 'none',
3326             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3327             '-mon', 'chardev=qmp,mode=control',
3328             '-pidfile', $pidfile,
3329             '-S', '-daemonize'
3330         ];
3331
3332         if (!$kvm) {
3333             push @$cmd, '-accel', 'tcg';
3334         }
3335
3336         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3337         die "QEMU flag querying VM exited with code " . $rc if $rc;
3338
3339         eval {
3340             my $cmd_result = mon_cmd(
3341                 $fakevmid,
3342                 'query-cpu-model-expansion',
3343                 type => 'full',
3344                 model => { name => 'host' }
3345             );
3346
3347             my $props = $cmd_result->{model}->{props};
3348             foreach my $prop (keys %$props) {
3349                 next if $props->{$prop} ne '1';
3350                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3351                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3352                 # We only keep those with underscores, to match /proc/cpuinfo
3353                 $prop =~ s/\.|-/_/g;
3354                 $flags->{$prop} = 1;
3355             }
3356         };
3357         my $err = $@;
3358
3359         # force stop with 10 sec timeout and 'nocheck'
3360         # always stop, even if QMP failed
3361         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3362
3363         die $err if $err;
3364
3365         return [ sort keys %$flags ];
3366     };
3367
3368     # We need to query QEMU twice, since KVM and TCG have different supported flags
3369     PVE::QemuConfig->lock_config($fakevmid, sub {
3370         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3371         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3372
3373         if ($kvm_supported) {
3374             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3375             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3376         }
3377     });
3378
3379     return $flags;
3380 }
3381
3382 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3383 my $understood_cpu_flag_dir = "/usr/share/kvm";
3384 sub query_understood_cpu_flags {
3385     my $arch = get_host_arch();
3386     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3387
3388     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3389         if ! -e $filepath;
3390
3391     my $raw = file_get_contents($filepath);
3392     $raw =~ s/^\s+|\s+$//g;
3393     my @flags = split(/\s+/, $raw);
3394
3395     return \@flags;
3396 }
3397
3398 sub config_to_command {
3399     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3400
3401     my $cmd = [];
3402     my $globalFlags = [];
3403     my $machineFlags = [];
3404     my $rtcFlags = [];
3405     my $devices = [];
3406     my $pciaddr = '';
3407     my $bridges = {};
3408     my $vernum = 0; # unknown
3409     my $ostype = $conf->{ostype};
3410     my $winversion = windows_version($ostype);
3411     my $kvm = $conf->{kvm};
3412     my $nodename = nodename();
3413
3414     my $arch = get_vm_arch($conf);
3415     my $kvm_binary = get_command_for_arch($arch);
3416     my $kvmver = kvm_user_version($kvm_binary);
3417
3418     my $add_pve_version = min_version($kvmver, 4, 1);
3419
3420     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3421     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3422     $kvm //= 1 if is_native($arch);
3423
3424     $machine_version =~ m/(\d+)\.(\d+)/;
3425     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3426         if !PVE::QemuServer::min_version($kvmver, $1, $2);
3427
3428     if ($kvm) {
3429         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3430             if !defined kvm_version();
3431     }
3432
3433     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3434         $vernum = $1*1000000+$2*1000;
3435     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3436         $vernum = $1*1000000+$2*1000+$3;
3437     }
3438
3439     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3440
3441     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3442     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3443     my $use_old_bios_files = undef;
3444     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3445
3446     my $cpuunits = defined($conf->{cpuunits}) ?
3447             $conf->{cpuunits} : $defaults->{cpuunits};
3448
3449     push @$cmd, $kvm_binary;
3450
3451     push @$cmd, '-id', $vmid;
3452
3453     my $vmname = $conf->{name} || "vm$vmid";
3454
3455     push @$cmd, '-name', $vmname;
3456
3457     my $use_virtio = 0;
3458
3459     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3460     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3461     push @$cmd, '-mon', "chardev=qmp,mode=control";
3462
3463     if (min_version($machine_version, 2, 12)) {
3464         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3465         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3466     }
3467
3468     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3469
3470     push @$cmd, '-daemonize';
3471
3472     if ($conf->{smbios1}) {
3473         my $smbios_conf = parse_smbios1($conf->{smbios1});
3474         if ($smbios_conf->{base64}) {
3475             # Do not pass base64 flag to qemu
3476             delete $smbios_conf->{base64};
3477             my $smbios_string = "";
3478             foreach my $key (keys %$smbios_conf) {
3479                 my $value;
3480                 if ($key eq "uuid") {
3481                     $value = $smbios_conf->{uuid}
3482                 } else {
3483                     $value = decode_base64($smbios_conf->{$key});
3484                 }
3485                 # qemu accepts any binary data, only commas need escaping by double comma
3486                 $value =~ s/,/,,/g;
3487                 $smbios_string .= "," . $key . "=" . $value if $value;
3488             }
3489             push @$cmd, '-smbios', "type=1" . $smbios_string;
3490         } else {
3491             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3492         }
3493     }
3494
3495     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3496     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3497         die "uefi base image not found\n" if ! -f $ovmf_code;
3498
3499         my $path;
3500         my $format;
3501         if (my $efidisk = $conf->{efidisk0}) {
3502             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3503             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3504             $format = $d->{format};
3505             if ($storeid) {
3506                 $path = PVE::Storage::path($storecfg, $d->{file});
3507                 if (!defined($format)) {
3508                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3509                     $format = qemu_img_format($scfg, $volname);
3510                 }
3511             } else {
3512                 $path = $d->{file};
3513                 die "efidisk format must be specified\n"
3514                     if !defined($format);
3515             }
3516         } else {
3517             warn "no efidisk configured! Using temporary efivars disk.\n";
3518             $path = "/tmp/$vmid-ovmf.fd";
3519             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3520             $format = 'raw';
3521         }
3522
3523         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3524         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3525     }
3526
3527     # load q35 config
3528     if ($q35) {
3529         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3530         if (min_version($machine_version, 4, 0)) {
3531             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3532         } else {
3533             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3534         }
3535     }
3536
3537     if ($conf->{vmgenid}) {
3538         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3539     }
3540
3541     # add usb controllers
3542     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3543     push @$devices, @usbcontrollers if @usbcontrollers;
3544     my $vga = parse_vga($conf->{vga});
3545
3546     my $qxlnum = vga_conf_has_spice($conf->{vga});
3547     $vga->{type} = 'qxl' if $qxlnum;
3548
3549     if (!$vga->{type}) {
3550         if ($arch eq 'aarch64') {
3551             $vga->{type} = 'virtio';
3552         } elsif (min_version($machine_version, 2, 9)) {
3553             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3554         } else {
3555             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3556         }
3557     }
3558
3559     # enable absolute mouse coordinates (needed by vnc)
3560     my $tablet;
3561     if (defined($conf->{tablet})) {
3562         $tablet = $conf->{tablet};
3563     } else {
3564         $tablet = $defaults->{tablet};
3565         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3566         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3567     }
3568
3569     if ($tablet) {
3570         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3571         my $kbd = print_keyboarddevice_full($conf, $arch);
3572         push @$devices, '-device', $kbd if defined($kbd);
3573     }
3574
3575     my $kvm_off = 0;
3576     my $gpu_passthrough;
3577
3578     # host pci devices
3579     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3580         my $id = "hostpci$i";
3581         my $d = parse_hostpci($conf->{$id});
3582         next if !$d;
3583
3584         if (my $pcie = $d->{pcie}) {
3585             die "q35 machine model is not enabled" if !$q35;
3586             # win7 wants to have the pcie devices directly on the pcie bus
3587             # instead of in the root port
3588             if ($winversion == 7) {
3589                 $pciaddr = print_pcie_addr("${id}bus0");
3590             } else {
3591                 # add more root ports if needed, 4 are present by default
3592                 # by pve-q35 cfgs, rest added here on demand.
3593                 if ($i > 3) {
3594                     push @$devices, '-device', print_pcie_root_port($i);
3595                 }
3596                 $pciaddr = print_pcie_addr($id);
3597             }
3598         } else {
3599             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3600         }
3601
3602         my $xvga = '';
3603         if ($d->{'x-vga'}) {
3604             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3605             $kvm_off = 1;
3606             $vga->{type} = 'none' if !defined($conf->{vga});
3607             $gpu_passthrough = 1;
3608         }
3609
3610         my $pcidevices = $d->{pciid};
3611         my $multifunction = 1 if @$pcidevices > 1;
3612
3613         my $sysfspath;
3614         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3615             my $pci_id = $pcidevices->[0]->{id};
3616             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3617             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3618         } elsif ($d->{mdev}) {
3619             warn "ignoring mediated device '$id' with multifunction device\n";
3620         }
3621
3622         my $j=0;
3623         foreach my $pcidevice (@$pcidevices) {
3624             my $devicestr = "vfio-pci";
3625
3626             if ($sysfspath) {
3627                 $devicestr .= ",sysfsdev=$sysfspath";
3628             } else {
3629                 $devicestr .= ",host=$pcidevice->{id}";
3630             }
3631
3632             my $mf_addr = $multifunction ? ".$j" : '';
3633             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3634
3635             if ($j == 0) {
3636                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3637                 $devicestr .= "$xvga";
3638                 $devicestr .= ",multifunction=on" if $multifunction;
3639                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3640             }
3641
3642             push @$devices, '-device', $devicestr;
3643             $j++;
3644         }
3645     }
3646
3647     # usb devices
3648     my $usb_dev_features = {};
3649     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3650
3651     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3652     push @$devices, @usbdevices if @usbdevices;
3653     # serial devices
3654     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3655         if (my $path = $conf->{"serial$i"}) {
3656             if ($path eq 'socket') {
3657                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3658                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3659                 # On aarch64, serial0 is the UART device. Qemu only allows
3660                 # connecting UART devices via the '-serial' command line, as
3661                 # the device has a fixed slot on the hardware...
3662                 if ($arch eq 'aarch64' && $i == 0) {
3663                     push @$devices, '-serial', "chardev:serial$i";
3664                 } else {
3665                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3666                 }
3667             } else {
3668                 die "no such serial device\n" if ! -c $path;
3669                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3670                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3671             }
3672         }
3673     }
3674
3675     # parallel devices
3676     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3677         if (my $path = $conf->{"parallel$i"}) {
3678             die "no such parallel device\n" if ! -c $path;
3679             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3680             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3681             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3682         }
3683     }
3684
3685     if (my $audio = conf_has_audio($conf)) {
3686
3687         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3688
3689         my $id = $audio->{dev_id};
3690         if ($audio->{dev} eq 'AC97') {
3691             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3692         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3693             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3694             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3695             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3696         } else {
3697             die "unkown audio device '$audio->{dev}', implement me!";
3698         }
3699
3700         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3701     }
3702
3703     my $sockets = 1;
3704     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3705     $sockets = $conf->{sockets} if  $conf->{sockets};
3706
3707     my $cores = $conf->{cores} || 1;
3708
3709     my $maxcpus = $sockets * $cores;
3710
3711     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3712
3713     my $allowed_vcpus = $cpuinfo->{cpus};
3714
3715     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3716         if ($allowed_vcpus < $maxcpus);
3717
3718     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3719
3720         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3721         for (my $i = 2; $i <= $vcpus; $i++)  {
3722             my $cpustr = print_cpu_device($conf,$i);
3723             push @$cmd, '-device', $cpustr;
3724         }
3725
3726     } else {
3727
3728         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3729     }
3730     push @$cmd, '-nodefaults';
3731
3732     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3733
3734     my $bootindex_hash = {};
3735     my $i = 1;
3736     foreach my $o (split(//, $bootorder)) {
3737         $bootindex_hash->{$o} = $i*100;
3738         $i++;
3739     }
3740
3741     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3742
3743     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3744
3745     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3746
3747     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3748         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3749         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3750         push @$cmd,  '-vnc', "unix:$socket,password";
3751     } else {
3752         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3753         push @$cmd, '-nographic';
3754     }
3755
3756     # time drift fix
3757     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3758
3759     my $useLocaltime = $conf->{localtime};
3760
3761     if ($winversion >= 5) { # windows
3762         $useLocaltime = 1 if !defined($conf->{localtime});
3763
3764         # use time drift fix when acpi is enabled
3765         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3766             $tdf = 1 if !defined($conf->{tdf});
3767         }
3768     }
3769
3770     if ($winversion >= 6) {
3771         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3772         push @$cmd, '-no-hpet';
3773     }
3774
3775     push @$rtcFlags, 'driftfix=slew' if $tdf;
3776
3777     if (!$kvm) {
3778         push @$machineFlags, 'accel=tcg';
3779     }
3780
3781     if ($machine_type) {
3782         push @$machineFlags, "type=${machine_type}";
3783     }
3784
3785     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3786         push @$rtcFlags, "base=$conf->{startdate}";
3787     } elsif ($useLocaltime) {
3788         push @$rtcFlags, 'base=localtime';
3789     }
3790
3791     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3792
3793     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3794
3795     push @$cmd, '-S' if $conf->{freeze};
3796
3797     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3798
3799     my $guest_agent = parse_guest_agent($conf);
3800
3801     if ($guest_agent->{enabled}) {
3802         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3803         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3804
3805         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3806             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3807             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3808             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3809         } elsif ($guest_agent->{type} eq 'isa') {
3810             push @$devices, '-device', "isa-serial,chardev=qga0";
3811         }
3812     }
3813
3814     my $spice_port;
3815
3816     if ($qxlnum) {
3817         if ($qxlnum > 1) {
3818             if ($winversion){
3819                 for(my $i = 1; $i < $qxlnum; $i++){
3820                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3821                 }
3822             } else {
3823                 # assume other OS works like Linux
3824                 my ($ram, $vram) = ("134217728", "67108864");
3825                 if ($vga->{memory}) {
3826                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3827                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3828                 }
3829                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3830                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3831             }
3832         }
3833
3834         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3835
3836         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3837         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3838         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3839
3840         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3841         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3842         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3843
3844         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3845         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3846
3847         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3848         if ($spice_enhancement->{foldersharing}) {
3849             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3850             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3851         }
3852
3853         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3854         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3855         push @$devices, '-spice', "$spice_opts";
3856     }
3857
3858     # enable balloon by default, unless explicitly disabled
3859     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3860         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3861         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3862     }
3863
3864     if ($conf->{watchdog}) {
3865         my $wdopts = parse_watchdog($conf->{watchdog});
3866         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3867         my $watchdog = $wdopts->{model} || 'i6300esb';
3868         push @$devices, '-device', "$watchdog$pciaddr";
3869         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3870     }
3871
3872     my $vollist = [];
3873     my $scsicontroller = {};
3874     my $ahcicontroller = {};
3875     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3876
3877     # Add iscsi initiator name if available
3878     if (my $initiator = get_initiator_name()) {
3879         push @$devices, '-iscsi', "initiator-name=$initiator";
3880     }
3881
3882     foreach_drive($conf, sub {
3883         my ($ds, $drive) = @_;
3884
3885         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3886             push @$vollist, $drive->{file};
3887         }
3888
3889         # ignore efidisk here, already added in bios/fw handling code above
3890         return if $drive->{interface} eq 'efidisk';
3891
3892         $use_virtio = 1 if $ds =~ m/^virtio/;
3893
3894         if (drive_is_cdrom ($drive)) {
3895             if ($bootindex_hash->{d}) {
3896                 $drive->{bootindex} = $bootindex_hash->{d};
3897                 $bootindex_hash->{d} += 1;
3898             }
3899         } else {
3900             if ($bootindex_hash->{c}) {
3901                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3902                 $bootindex_hash->{c} += 1;
3903             }
3904         }
3905
3906         if($drive->{interface} eq 'virtio'){
3907            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3908         }
3909
3910         if ($drive->{interface} eq 'scsi') {
3911
3912             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3913
3914             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3915             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3916
3917             my $iothread = '';
3918             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3919                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3920                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3921             } elsif ($drive->{iothread}) {
3922                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3923             }
3924
3925             my $queues = '';
3926             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3927                 $queues = ",num_queues=$drive->{queues}";
3928             }
3929
3930             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3931             $scsicontroller->{$controller}=1;
3932         }
3933
3934         if ($drive->{interface} eq 'sata') {
3935            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3936            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3937            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3938            $ahcicontroller->{$controller}=1;
3939         }
3940
3941         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3942         push @$devices, '-drive',$drive_cmd;
3943         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3944     });
3945
3946     for (my $i = 0; $i < $MAX_NETS; $i++) {
3947          next if !$conf->{"net$i"};
3948          my $d = parse_net($conf->{"net$i"});
3949          next if !$d;
3950
3951          $use_virtio = 1 if $d->{model} eq 'virtio';
3952
3953          if ($bootindex_hash->{n}) {
3954             $d->{bootindex} = $bootindex_hash->{n};
3955             $bootindex_hash->{n} += 1;
3956          }
3957
3958          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3959          push @$devices, '-netdev', $netdevfull;
3960
3961          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3962          push @$devices, '-device', $netdevicefull;
3963     }
3964
3965     if ($conf->{ivshmem}) {
3966         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3967
3968         my $bus;
3969         if ($q35) {
3970             $bus = print_pcie_addr("ivshmem");
3971         } else {
3972             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3973         }
3974
3975         my $ivshmem_name = $ivshmem->{name} // $vmid;
3976         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3977
3978         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3979         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3980     }
3981
3982     if (!$q35) {
3983         # add pci bridges
3984         if (min_version($machine_version, 2, 3)) {
3985            $bridges->{1} = 1;
3986            $bridges->{2} = 1;
3987         }
3988
3989         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3990
3991         for my $k (sort {$b cmp $a} keys %$bridges) {
3992             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3993             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3994         }
3995     }
3996
3997     push @$cmd, @$devices;
3998     push @$cmd, '-rtc', join(',', @$rtcFlags)
3999         if scalar(@$rtcFlags);
4000     push @$cmd, '-machine', join(',', @$machineFlags)
4001         if scalar(@$machineFlags);
4002     push @$cmd, '-global', join(',', @$globalFlags)
4003         if scalar(@$globalFlags);
4004
4005     if (my $vmstate = $conf->{vmstate}) {
4006         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4007         push @$vollist, $vmstate;
4008         push @$cmd, '-loadstate', $statepath;
4009         print "activating and using '$vmstate' as vmstate\n";
4010     }
4011
4012     # add custom args
4013     if ($conf->{args}) {
4014         my $aa = PVE::Tools::split_args($conf->{args});
4015         push @$cmd, @$aa;
4016     }
4017
4018     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4019 }
4020
4021 sub spice_port {
4022     my ($vmid) = @_;
4023
4024     my $res = mon_cmd($vmid, 'query-spice');
4025
4026     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4027 }
4028
4029 sub vm_devices_list {
4030     my ($vmid) = @_;
4031
4032     my $res = mon_cmd($vmid, 'query-pci');
4033     my $devices_to_check = [];
4034     my $devices = {};
4035     foreach my $pcibus (@$res) {
4036         push @$devices_to_check, @{$pcibus->{devices}},
4037     }
4038
4039     while (@$devices_to_check) {
4040         my $to_check = [];
4041         for my $d (@$devices_to_check) {
4042             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4043             next if !$d->{'pci_bridge'};
4044
4045             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4046             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4047         }
4048         $devices_to_check = $to_check;
4049     }
4050
4051     my $resblock = mon_cmd($vmid, 'query-block');
4052     foreach my $block (@$resblock) {
4053         if($block->{device} =~ m/^drive-(\S+)/){
4054                 $devices->{$1} = 1;
4055         }
4056     }
4057
4058     my $resmice = mon_cmd($vmid, 'query-mice');
4059     foreach my $mice (@$resmice) {
4060         if ($mice->{name} eq 'QEMU HID Tablet') {
4061             $devices->{tablet} = 1;
4062             last;
4063         }
4064     }
4065
4066     # for usb devices there is no query-usb
4067     # but we can iterate over the entries in
4068     # qom-list path=/machine/peripheral
4069     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4070     foreach my $per (@$resperipheral) {
4071         if ($per->{name} =~ m/^usb\d+$/) {
4072             $devices->{$per->{name}} = 1;
4073         }
4074     }
4075
4076     return $devices;
4077 }
4078
4079 sub vm_deviceplug {
4080     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4081
4082     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4083
4084     my $devices_list = vm_devices_list($vmid);
4085     return 1 if defined($devices_list->{$deviceid});
4086
4087     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4088
4089     if ($deviceid eq 'tablet') {
4090
4091         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4092
4093     } elsif ($deviceid eq 'keyboard') {
4094
4095         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4096
4097     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4098
4099         die "usb hotplug currently not reliable\n";
4100         # since we can't reliably hot unplug all added usb devices
4101         # and usb passthrough disables live migration
4102         # we disable usb hotplugging for now
4103         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4104
4105     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4106
4107         qemu_iothread_add($vmid, $deviceid, $device);
4108
4109         qemu_driveadd($storecfg, $vmid, $device);
4110         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4111
4112         qemu_deviceadd($vmid, $devicefull);
4113         eval { qemu_deviceaddverify($vmid, $deviceid); };
4114         if (my $err = $@) {
4115             eval { qemu_drivedel($vmid, $deviceid); };
4116             warn $@ if $@;
4117             die $err;
4118         }
4119
4120     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4121
4122
4123         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4124         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4125         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4126
4127         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4128
4129         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4130             qemu_iothread_add($vmid, $deviceid, $device);
4131             $devicefull .= ",iothread=iothread-$deviceid";
4132         }
4133
4134         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4135             $devicefull .= ",num_queues=$device->{queues}";
4136         }
4137
4138         qemu_deviceadd($vmid, $devicefull);
4139         qemu_deviceaddverify($vmid, $deviceid);
4140
4141     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4142
4143         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4144         qemu_driveadd($storecfg, $vmid, $device);
4145
4146         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4147         eval { qemu_deviceadd($vmid, $devicefull); };
4148         if (my $err = $@) {
4149             eval { qemu_drivedel($vmid, $deviceid); };
4150             warn $@ if $@;
4151             die $err;
4152         }
4153
4154     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4155
4156         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4157
4158         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4159         my $use_old_bios_files = undef;
4160         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4161
4162         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4163         qemu_deviceadd($vmid, $netdevicefull);
4164         eval {
4165             qemu_deviceaddverify($vmid, $deviceid);
4166             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4167         };
4168         if (my $err = $@) {
4169             eval { qemu_netdevdel($vmid, $deviceid); };
4170             warn $@ if $@;
4171             die $err;
4172         }
4173
4174     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4175
4176         my $bridgeid = $2;
4177         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4178         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4179
4180         qemu_deviceadd($vmid, $devicefull);
4181         qemu_deviceaddverify($vmid, $deviceid);
4182
4183     } else {
4184         die "can't hotplug device '$deviceid'\n";
4185     }
4186
4187     return 1;
4188 }
4189
4190 # fixme: this should raise exceptions on error!
4191 sub vm_deviceunplug {
4192     my ($vmid, $conf, $deviceid) = @_;
4193
4194     my $devices_list = vm_devices_list($vmid);
4195     return 1 if !defined($devices_list->{$deviceid});
4196
4197     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4198
4199     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4200
4201         qemu_devicedel($vmid, $deviceid);
4202
4203     } elsif ($deviceid =~ m/^usb\d+$/) {
4204
4205         die "usb hotplug currently not reliable\n";
4206         # when unplugging usb devices this way,
4207         # there may be remaining usb controllers/hubs
4208         # so we disable it for now
4209         qemu_devicedel($vmid, $deviceid);
4210         qemu_devicedelverify($vmid, $deviceid);
4211
4212     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4213
4214         qemu_devicedel($vmid, $deviceid);
4215         qemu_devicedelverify($vmid, $deviceid);
4216         qemu_drivedel($vmid, $deviceid);
4217         qemu_iothread_del($conf, $vmid, $deviceid);
4218
4219     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4220
4221         qemu_devicedel($vmid, $deviceid);
4222         qemu_devicedelverify($vmid, $deviceid);
4223         qemu_iothread_del($conf, $vmid, $deviceid);
4224
4225     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4226
4227         qemu_devicedel($vmid, $deviceid);
4228         qemu_drivedel($vmid, $deviceid);
4229         qemu_deletescsihw($conf, $vmid, $deviceid);
4230
4231     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4232
4233         qemu_devicedel($vmid, $deviceid);
4234         qemu_devicedelverify($vmid, $deviceid);
4235         qemu_netdevdel($vmid, $deviceid);
4236
4237     } else {
4238         die "can't unplug device '$deviceid'\n";
4239     }
4240
4241     return 1;
4242 }
4243
4244 sub qemu_deviceadd {
4245     my ($vmid, $devicefull) = @_;
4246
4247     $devicefull = "driver=".$devicefull;
4248     my %options =  split(/[=,]/, $devicefull);
4249
4250     mon_cmd($vmid, "device_add" , %options);
4251 }
4252
4253 sub qemu_devicedel {
4254     my ($vmid, $deviceid) = @_;
4255
4256     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4257 }
4258
4259 sub qemu_iothread_add {
4260     my($vmid, $deviceid, $device) = @_;
4261
4262     if ($device->{iothread}) {
4263         my $iothreads = vm_iothreads_list($vmid);
4264         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4265     }
4266 }
4267
4268 sub qemu_iothread_del {
4269     my($conf, $vmid, $deviceid) = @_;
4270
4271     my $confid = $deviceid;
4272     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4273         $confid = 'scsi' . $1;
4274     }
4275     my $device = parse_drive($confid, $conf->{$confid});
4276     if ($device->{iothread}) {
4277         my $iothreads = vm_iothreads_list($vmid);
4278         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4279     }
4280 }
4281
4282 sub qemu_objectadd {
4283     my($vmid, $objectid, $qomtype) = @_;
4284
4285     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4286
4287     return 1;
4288 }
4289
4290 sub qemu_objectdel {
4291     my($vmid, $objectid) = @_;
4292
4293     mon_cmd($vmid, "object-del", id => $objectid);
4294
4295     return 1;
4296 }
4297
4298 sub qemu_driveadd {
4299     my ($storecfg, $vmid, $device) = @_;
4300
4301     my $drive = print_drive_full($storecfg, $vmid, $device);
4302     $drive =~ s/\\/\\\\/g;
4303     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4304
4305     # If the command succeeds qemu prints: "OK"
4306     return 1 if $ret =~ m/OK/s;
4307
4308     die "adding drive failed: $ret\n";
4309 }
4310
4311 sub qemu_drivedel {
4312     my($vmid, $deviceid) = @_;
4313
4314     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4315     $ret =~ s/^\s+//;
4316
4317     return 1 if $ret eq "";
4318
4319     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4320     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4321
4322     die "deleting drive $deviceid failed : $ret\n";
4323 }
4324
4325 sub qemu_deviceaddverify {
4326     my ($vmid, $deviceid) = @_;
4327
4328     for (my $i = 0; $i <= 5; $i++) {
4329          my $devices_list = vm_devices_list($vmid);
4330          return 1 if defined($devices_list->{$deviceid});
4331          sleep 1;
4332     }
4333
4334     die "error on hotplug device '$deviceid'\n";
4335 }
4336
4337
4338 sub qemu_devicedelverify {
4339     my ($vmid, $deviceid) = @_;
4340
4341     # need to verify that the device is correctly removed as device_del
4342     # is async and empty return is not reliable
4343
4344     for (my $i = 0; $i <= 5; $i++) {
4345          my $devices_list = vm_devices_list($vmid);
4346          return 1 if !defined($devices_list->{$deviceid});
4347          sleep 1;
4348     }
4349
4350     die "error on hot-unplugging device '$deviceid'\n";
4351 }
4352
4353 sub qemu_findorcreatescsihw {
4354     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4355
4356     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4357
4358     my $scsihwid="$controller_prefix$controller";
4359     my $devices_list = vm_devices_list($vmid);
4360
4361     if(!defined($devices_list->{$scsihwid})) {
4362         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4363     }
4364
4365     return 1;
4366 }
4367
4368 sub qemu_deletescsihw {
4369     my ($conf, $vmid, $opt) = @_;
4370
4371     my $device = parse_drive($opt, $conf->{$opt});
4372
4373     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4374         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4375         return 1;
4376     }
4377
4378     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4379
4380     my $devices_list = vm_devices_list($vmid);
4381     foreach my $opt (keys %{$devices_list}) {
4382         if (PVE::QemuServer::is_valid_drivename($opt)) {
4383             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4384             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4385                 return 1;
4386             }
4387         }
4388     }
4389
4390     my $scsihwid="scsihw$controller";
4391
4392     vm_deviceunplug($vmid, $conf, $scsihwid);
4393
4394     return 1;
4395 }
4396
4397 sub qemu_add_pci_bridge {
4398     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4399
4400     my $bridges = {};
4401
4402     my $bridgeid;
4403
4404     print_pci_addr($device, $bridges, $arch, $machine_type);
4405
4406     while (my ($k, $v) = each %$bridges) {
4407         $bridgeid = $k;
4408     }
4409     return 1 if !defined($bridgeid) || $bridgeid < 1;
4410
4411     my $bridge = "pci.$bridgeid";
4412     my $devices_list = vm_devices_list($vmid);
4413
4414     if (!defined($devices_list->{$bridge})) {
4415         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4416     }
4417
4418     return 1;
4419 }
4420
4421 sub qemu_set_link_status {
4422     my ($vmid, $device, $up) = @_;
4423
4424     mon_cmd($vmid, "set_link", name => $device,
4425                up => $up ? JSON::true : JSON::false);
4426 }
4427
4428 sub qemu_netdevadd {
4429     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4430
4431     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4432     my %options =  split(/[=,]/, $netdev);
4433
4434     mon_cmd($vmid, "netdev_add",  %options);
4435     return 1;
4436 }
4437
4438 sub qemu_netdevdel {
4439     my ($vmid, $deviceid) = @_;
4440
4441     mon_cmd($vmid, "netdev_del", id => $deviceid);
4442 }
4443
4444 sub qemu_usb_hotplug {
4445     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4446
4447     return if !$device;
4448
4449     # remove the old one first
4450     vm_deviceunplug($vmid, $conf, $deviceid);
4451
4452     # check if xhci controller is necessary and available
4453     if ($device->{usb3}) {
4454
4455         my $devicelist = vm_devices_list($vmid);
4456
4457         if (!$devicelist->{xhci}) {
4458             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4459             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4460         }
4461     }
4462     my $d = parse_usb_device($device->{host});
4463     $d->{usb3} = $device->{usb3};
4464
4465     # add the new one
4466     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4467 }
4468
4469 sub qemu_cpu_hotplug {
4470     my ($vmid, $conf, $vcpus) = @_;
4471
4472     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4473
4474     my $sockets = 1;
4475     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4476     $sockets = $conf->{sockets} if  $conf->{sockets};
4477     my $cores = $conf->{cores} || 1;
4478     my $maxcpus = $sockets * $cores;
4479
4480     $vcpus = $maxcpus if !$vcpus;
4481
4482     die "you can't add more vcpus than maxcpus\n"
4483         if $vcpus > $maxcpus;
4484
4485     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4486
4487     if ($vcpus < $currentvcpus) {
4488
4489         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4490
4491             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4492                 qemu_devicedel($vmid, "cpu$i");
4493                 my $retry = 0;
4494                 my $currentrunningvcpus = undef;
4495                 while (1) {
4496                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4497                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4498                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4499                     $retry++;
4500                     sleep 1;
4501                 }
4502                 #update conf after each succesfull cpu unplug
4503                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4504                 PVE::QemuConfig->write_config($vmid, $conf);
4505             }
4506         } else {
4507             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4508         }
4509
4510         return;
4511     }
4512
4513     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4514     die "vcpus in running vm does not match its configuration\n"
4515         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4516
4517     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4518
4519         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4520             my $cpustr = print_cpu_device($conf, $i);
4521             qemu_deviceadd($vmid, $cpustr);
4522
4523             my $retry = 0;
4524             my $currentrunningvcpus = undef;
4525             while (1) {
4526                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4527                 last if scalar(@{$currentrunningvcpus}) == $i;
4528                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4529                 sleep 1;
4530                 $retry++;
4531             }
4532             #update conf after each succesfull cpu hotplug
4533             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4534             PVE::QemuConfig->write_config($vmid, $conf);
4535         }
4536     } else {
4537
4538         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4539             mon_cmd($vmid, "cpu-add", id => int($i));
4540         }
4541     }
4542 }
4543
4544 sub qemu_block_set_io_throttle {
4545     my ($vmid, $deviceid,
4546         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4547         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4548         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4549         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4550
4551     return if !check_running($vmid) ;
4552
4553     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4554         bps => int($bps),
4555         bps_rd => int($bps_rd),
4556         bps_wr => int($bps_wr),
4557         iops => int($iops),
4558         iops_rd => int($iops_rd),
4559         iops_wr => int($iops_wr),
4560         bps_max => int($bps_max),
4561         bps_rd_max => int($bps_rd_max),
4562         bps_wr_max => int($bps_wr_max),
4563         iops_max => int($iops_max),
4564         iops_rd_max => int($iops_rd_max),
4565         iops_wr_max => int($iops_wr_max),
4566         bps_max_length => int($bps_max_length),
4567         bps_rd_max_length => int($bps_rd_max_length),
4568         bps_wr_max_length => int($bps_wr_max_length),
4569         iops_max_length => int($iops_max_length),
4570         iops_rd_max_length => int($iops_rd_max_length),
4571         iops_wr_max_length => int($iops_wr_max_length),
4572     );
4573
4574 }
4575
4576 # old code, only used to shutdown old VM after update
4577 sub __read_avail {
4578     my ($fh, $timeout) = @_;
4579
4580     my $sel = new IO::Select;
4581     $sel->add($fh);
4582
4583     my $res = '';
4584     my $buf;
4585
4586     my @ready;
4587     while (scalar (@ready = $sel->can_read($timeout))) {
4588         my $count;
4589         if ($count = $fh->sysread($buf, 8192)) {
4590             if ($buf =~ /^(.*)\(qemu\) $/s) {
4591                 $res .= $1;
4592                 last;
4593             } else {
4594                 $res .= $buf;
4595             }
4596         } else {
4597             if (!defined($count)) {
4598                 die "$!\n";
4599             }
4600             last;
4601         }
4602     }
4603
4604     die "monitor read timeout\n" if !scalar(@ready);
4605
4606     return $res;
4607 }
4608
4609 sub qemu_block_resize {
4610     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4611
4612     my $running = check_running($vmid);
4613
4614     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4615
4616     return if !$running;
4617
4618     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4619
4620 }
4621
4622 sub qemu_volume_snapshot {
4623     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4624
4625     my $running = check_running($vmid);
4626
4627     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4628         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4629     } else {
4630         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4631     }
4632 }
4633
4634 sub qemu_volume_snapshot_delete {
4635     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4636
4637     my $running = check_running($vmid);
4638
4639     if($running) {
4640
4641         $running = undef;
4642         my $conf = PVE::QemuConfig->load_config($vmid);
4643         foreach_drive($conf, sub {
4644             my ($ds, $drive) = @_;
4645             $running = 1 if $drive->{file} eq $volid;
4646         });
4647     }
4648
4649     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4650         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4651     } else {
4652         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4653     }
4654 }
4655
4656 sub set_migration_caps {
4657     my ($vmid) = @_;
4658
4659     my $cap_ref = [];
4660
4661     my $enabled_cap = {
4662         "auto-converge" => 1,
4663         "xbzrle" => 1,
4664         "x-rdma-pin-all" => 0,
4665         "zero-blocks" => 0,
4666         "compress" => 0
4667     };
4668
4669     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4670
4671     for my $supported_capability (@$supported_capabilities) {
4672         push @$cap_ref, {
4673             capability => $supported_capability->{capability},
4674             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4675         };
4676     }
4677
4678     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4679 }
4680
4681 my $fast_plug_option = {
4682     'lock' => 1,
4683     'name' => 1,
4684     'onboot' => 1,
4685     'shares' => 1,
4686     'startup' => 1,
4687     'description' => 1,
4688     'protection' => 1,
4689     'vmstatestorage' => 1,
4690     'hookscript' => 1,
4691     'tags' => 1,
4692 };
4693
4694 # hotplug changes in [PENDING]
4695 # $selection hash can be used to only apply specified options, for
4696 # example: { cores => 1 } (only apply changed 'cores')
4697 # $errors ref is used to return error messages
4698 sub vmconfig_hotplug_pending {
4699     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4700
4701     my $defaults = load_defaults();
4702     my $arch = get_vm_arch($conf);
4703     my $machine_type = get_vm_machine($conf, undef, $arch);
4704
4705     # commit values which do not have any impact on running VM first
4706     # Note: those option cannot raise errors, we we do not care about
4707     # $selection and always apply them.
4708
4709     my $add_error = sub {
4710         my ($opt, $msg) = @_;
4711         $errors->{$opt} = "hotplug problem - $msg";
4712     };
4713
4714     my $changes = 0;
4715     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4716         if ($fast_plug_option->{$opt}) {
4717             $conf->{$opt} = $conf->{pending}->{$opt};
4718             delete $conf->{pending}->{$opt};
4719             $changes = 1;
4720         }
4721     }
4722
4723     if ($changes) {
4724         PVE::QemuConfig->write_config($vmid, $conf);
4725     }
4726
4727     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4728
4729     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4730     foreach my $opt (sort keys %$pending_delete_hash) {
4731         next if $selection && !$selection->{$opt};
4732         my $force = $pending_delete_hash->{$opt}->{force};
4733         eval {
4734             if ($opt eq 'hotplug') {
4735                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4736             } elsif ($opt eq 'tablet') {
4737                 die "skip\n" if !$hotplug_features->{usb};
4738                 if ($defaults->{tablet}) {
4739                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4740                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4741                         if $arch eq 'aarch64';
4742                 } else {
4743                     vm_deviceunplug($vmid, $conf, 'tablet');
4744                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4745                 }
4746             } elsif ($opt =~ m/^usb\d+/) {
4747                 die "skip\n";
4748                 # since we cannot reliably hot unplug usb devices
4749                 # we are disabling it
4750                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4751                 vm_deviceunplug($vmid, $conf, $opt);
4752             } elsif ($opt eq 'vcpus') {
4753                 die "skip\n" if !$hotplug_features->{cpu};
4754                 qemu_cpu_hotplug($vmid, $conf, undef);
4755             } elsif ($opt eq 'balloon') {
4756                 # enable balloon device is not hotpluggable
4757                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4758                 # here we reset the ballooning value to memory
4759                 my $balloon = $conf->{memory} || $defaults->{memory};
4760                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4761             } elsif ($fast_plug_option->{$opt}) {
4762                 # do nothing
4763             } elsif ($opt =~ m/^net(\d+)$/) {
4764                 die "skip\n" if !$hotplug_features->{network};
4765                 vm_deviceunplug($vmid, $conf, $opt);
4766             } elsif (is_valid_drivename($opt)) {
4767                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4768                 vm_deviceunplug($vmid, $conf, $opt);
4769                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4770             } elsif ($opt =~ m/^memory$/) {
4771                 die "skip\n" if !$hotplug_features->{memory};
4772                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4773             } elsif ($opt eq 'cpuunits') {
4774                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4775             } elsif ($opt eq 'cpulimit') {
4776                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4777             } else {
4778                 die "skip\n";
4779             }
4780         };
4781         if (my $err = $@) {
4782             &$add_error($opt, $err) if $err ne "skip\n";
4783         } else {
4784             delete $conf->{$opt};
4785             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4786         }
4787     }
4788
4789     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4790     $apply_pending_cloudinit = sub {
4791         return if $apply_pending_cloudinit_done; # once is enough
4792         $apply_pending_cloudinit_done = 1; # once is enough
4793
4794         my ($key, $value) = @_;
4795
4796         my @cloudinit_opts = keys %$confdesc_cloudinit;
4797         foreach my $opt (keys %{$conf->{pending}}) {
4798             next if !grep { $_ eq $opt } @cloudinit_opts;
4799             $conf->{$opt} = delete $conf->{pending}->{$opt};
4800         }
4801
4802         my $new_conf = { %$conf };
4803         $new_conf->{$key} = $value;
4804         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4805     };
4806
4807     foreach my $opt (keys %{$conf->{pending}}) {
4808         next if $selection && !$selection->{$opt};
4809         my $value = $conf->{pending}->{$opt};
4810         eval {
4811             if ($opt eq 'hotplug') {
4812                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4813             } elsif ($opt eq 'tablet') {
4814                 die "skip\n" if !$hotplug_features->{usb};
4815                 if ($value == 1) {
4816                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4817                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4818                         if $arch eq 'aarch64';
4819                 } elsif ($value == 0) {
4820                     vm_deviceunplug($vmid, $conf, 'tablet');
4821                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4822                 }
4823             } elsif ($opt =~ m/^usb\d+$/) {
4824                 die "skip\n";
4825                 # since we cannot reliably hot unplug usb devices
4826                 # we are disabling it
4827                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4828                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4829                 die "skip\n" if !$d;
4830                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4831             } elsif ($opt eq 'vcpus') {
4832                 die "skip\n" if !$hotplug_features->{cpu};
4833                 qemu_cpu_hotplug($vmid, $conf, $value);
4834             } elsif ($opt eq 'balloon') {
4835                 # enable/disable balloning device is not hotpluggable
4836                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4837                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4838                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4839
4840                 # allow manual ballooning if shares is set to zero
4841                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4842                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4843                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4844                 }
4845             } elsif ($opt =~ m/^net(\d+)$/) {
4846                 # some changes can be done without hotplug
4847                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4848                                     $vmid, $opt, $value, $arch, $machine_type);
4849             } elsif (is_valid_drivename($opt)) {
4850                 die "skip\n" if $opt eq 'efidisk0';
4851                 # some changes can be done without hotplug
4852                 my $drive = parse_drive($opt, $value);
4853                 if (drive_is_cloudinit($drive)) {
4854                     &$apply_pending_cloudinit($opt, $value);
4855                 }
4856                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4857                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4858             } elsif ($opt =~ m/^memory$/) { #dimms
4859                 die "skip\n" if !$hotplug_features->{memory};
4860                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4861             } elsif ($opt eq 'cpuunits') {
4862                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4863             } elsif ($opt eq 'cpulimit') {
4864                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4865                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4866             } else {
4867                 die "skip\n";  # skip non-hot-pluggable options
4868             }
4869         };
4870         if (my $err = $@) {
4871             &$add_error($opt, $err) if $err ne "skip\n";
4872         } else {
4873             $conf->{$opt} = $value;
4874             delete $conf->{pending}->{$opt};
4875         }
4876     }
4877
4878     PVE::QemuConfig->write_config($vmid, $conf);
4879 }
4880
4881 sub try_deallocate_drive {
4882     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4883
4884     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4885         my $volid = $drive->{file};
4886         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4887             my $sid = PVE::Storage::parse_volume_id($volid);
4888             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4889
4890             # check if the disk is really unused
4891             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4892                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4893             PVE::Storage::vdisk_free($storecfg, $volid);
4894             return 1;
4895         } else {
4896             # If vm is not owner of this disk remove from config
4897             return 1;
4898         }
4899     }
4900
4901     return undef;
4902 }
4903
4904 sub vmconfig_delete_or_detach_drive {
4905     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4906
4907     my $drive = parse_drive($opt, $conf->{$opt});
4908
4909     my $rpcenv = PVE::RPCEnvironment::get();
4910     my $authuser = $rpcenv->get_user();
4911
4912     if ($force) {
4913         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4914         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4915     } else {
4916         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4917     }
4918 }
4919
4920
4921
4922 sub vmconfig_apply_pending {
4923     my ($vmid, $conf, $storecfg, $errors) = @_;
4924
4925     my $add_apply_error = sub {
4926         my ($opt, $msg) = @_;
4927         my $err_msg = "unable to apply pending change $opt : $msg";
4928         $errors->{$opt} = $err_msg;
4929         warn $err_msg;
4930     };
4931
4932     # cold plug
4933
4934     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4935     foreach my $opt (sort keys %$pending_delete_hash) {
4936         my $force = $pending_delete_hash->{$opt}->{force};
4937         eval {
4938             if ($opt =~ m/^unused/) {
4939                 die "internal error";
4940             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4941                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4942             }
4943         };
4944         if (my $err = $@) {
4945             $add_apply_error->($opt, $err);
4946         } else {
4947             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4948             delete $conf->{$opt};
4949         }
4950     }
4951
4952     PVE::QemuConfig->cleanup_pending($conf);
4953
4954     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4955         next if $opt eq 'delete'; # just to be sure
4956         eval {
4957             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4958                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4959             }
4960         };
4961         if (my $err = $@) {
4962             $add_apply_error->($opt, $err);
4963         } else {
4964             $conf->{$opt} = delete $conf->{pending}->{$opt};
4965         }
4966     }
4967
4968     # write all changes at once to avoid unnecessary i/o
4969     PVE::QemuConfig->write_config($vmid, $conf);
4970 }
4971
4972 my $safe_num_ne = sub {
4973     my ($a, $b) = @_;
4974
4975     return 0 if !defined($a) && !defined($b);
4976     return 1 if !defined($a);
4977     return 1 if !defined($b);
4978
4979     return $a != $b;
4980 };
4981
4982 my $safe_string_ne = sub {
4983     my ($a, $b) = @_;
4984
4985     return 0 if !defined($a) && !defined($b);
4986     return 1 if !defined($a);
4987     return 1 if !defined($b);
4988
4989     return $a ne $b;
4990 };
4991
4992 sub vmconfig_update_net {
4993     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4994
4995     my $newnet = parse_net($value);
4996
4997     if ($conf->{$opt}) {
4998         my $oldnet = parse_net($conf->{$opt});
4999
5000         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5001             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5002             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5003             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5004
5005             # for non online change, we try to hot-unplug
5006             die "skip\n" if !$hotplug;
5007             vm_deviceunplug($vmid, $conf, $opt);
5008         } else {
5009
5010             die "internal error" if $opt !~ m/net(\d+)/;
5011             my $iface = "tap${vmid}i$1";
5012
5013             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5014                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5015                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5016                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5017                 PVE::Network::tap_unplug($iface);
5018                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5019             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5020                 # Rate can be applied on its own but any change above needs to
5021                 # include the rate in tap_plug since OVS resets everything.
5022                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5023             }
5024
5025             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5026                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5027             }
5028
5029             return 1;
5030         }
5031     }
5032
5033     if ($hotplug) {
5034         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5035     } else {
5036         die "skip\n";
5037     }
5038 }
5039
5040 sub vmconfig_update_disk {
5041     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5042
5043     # fixme: do we need force?
5044
5045     my $drive = parse_drive($opt, $value);
5046
5047     if ($conf->{$opt}) {
5048
5049         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5050
5051             my $media = $drive->{media} || 'disk';
5052             my $oldmedia = $old_drive->{media} || 'disk';
5053             die "unable to change media type\n" if $media ne $oldmedia;
5054
5055             if (!drive_is_cdrom($old_drive)) {
5056
5057                 if ($drive->{file} ne $old_drive->{file}) {
5058
5059                     die "skip\n" if !$hotplug;
5060
5061                     # unplug and register as unused
5062                     vm_deviceunplug($vmid, $conf, $opt);
5063                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5064
5065                 } else {
5066                     # update existing disk
5067
5068                     # skip non hotpluggable value
5069                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5070                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5071                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5072                         &$safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5073                         &$safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5074                         die "skip\n";
5075                     }
5076
5077                     # apply throttle
5078                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5079                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5080                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5081                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5082                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5083                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5084                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5085                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5086                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5087                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5088                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5089                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5090                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5091                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5092                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5093                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5094                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5095                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5096
5097                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5098                                                    ($drive->{mbps} || 0)*1024*1024,
5099                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5100                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5101                                                    $drive->{iops} || 0,
5102                                                    $drive->{iops_rd} || 0,
5103                                                    $drive->{iops_wr} || 0,
5104                                                    ($drive->{mbps_max} || 0)*1024*1024,
5105                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5106                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5107                                                    $drive->{iops_max} || 0,
5108                                                    $drive->{iops_rd_max} || 0,
5109                                                    $drive->{iops_wr_max} || 0,
5110                                                    $drive->{bps_max_length} || 1,
5111                                                    $drive->{bps_rd_max_length} || 1,
5112                                                    $drive->{bps_wr_max_length} || 1,
5113                                                    $drive->{iops_max_length} || 1,
5114                                                    $drive->{iops_rd_max_length} || 1,
5115                                                    $drive->{iops_wr_max_length} || 1);
5116
5117                     }
5118
5119                     return 1;
5120                 }
5121
5122             } else { # cdrom
5123
5124                 if ($drive->{file} eq 'none') {
5125                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5126                     if (drive_is_cloudinit($old_drive)) {
5127                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5128                     }
5129                 } else {
5130                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5131                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5132                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5133                 }
5134
5135                 return 1;
5136             }
5137         }
5138     }
5139
5140     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5141     # hotplug new disks
5142     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5143     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5144 }
5145
5146 sub vm_start {
5147     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5148         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage, $timeout) = @_;
5149
5150     PVE::QemuConfig->lock_config($vmid, sub {
5151         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5152
5153         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5154
5155         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5156
5157         PVE::QemuConfig->check_lock($conf)
5158             if !($skiplock || $is_suspended);
5159
5160         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5161
5162         # clean up leftover reboot request files
5163         eval { clear_reboot_request($vmid); };
5164         warn $@ if $@;
5165
5166         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5167             vmconfig_apply_pending($vmid, $conf, $storecfg);
5168             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5169         }
5170
5171         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5172
5173         my $defaults = load_defaults();
5174
5175         # set environment variable useful inside network script
5176         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5177
5178         my $local_volumes = {};
5179
5180         if ($targetstorage) {
5181             foreach_drive($conf, sub {
5182                 my ($ds, $drive) = @_;
5183
5184                 return if drive_is_cdrom($drive);
5185
5186                 my $volid = $drive->{file};
5187
5188                 return if !$volid;
5189
5190                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5191
5192                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5193                 return if $scfg->{shared};
5194                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5195             });
5196
5197             my $format = undef;
5198
5199             foreach my $opt (sort keys %$local_volumes) {
5200
5201                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5202                 my $drive = parse_drive($opt, $conf->{$opt});
5203
5204                 # If a remote storage is specified and the format of the original
5205                 # volume is not available there, fall back to the default format.
5206                 # Otherwise use the same format as the original.
5207                 if ($targetstorage && $targetstorage ne "1") {
5208                     $storeid = $targetstorage;
5209                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5210                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5211                     my $fileFormat = qemu_img_format($scfg, $volname);
5212                     $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5213                 } else {
5214                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5215                     $format = qemu_img_format($scfg, $volname);
5216                 }
5217
5218                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5219                 my $newdrive = $drive;
5220                 $newdrive->{format} = $format;
5221                 $newdrive->{file} = $newvolid;
5222                 my $drivestr = print_drive($newdrive);
5223                 $local_volumes->{$opt} = $drivestr;
5224                 #pass drive to conf for command line
5225                 $conf->{$opt} = $drivestr;
5226             }
5227         }
5228
5229         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5230
5231         if ($is_suspended) {
5232             # enforce machine type on suspended vm to ensure HW compatibility
5233             $forcemachine = $conf->{runningmachine};
5234             print "Resuming suspended VM\n";
5235         }
5236
5237         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5238
5239         my $migration_ip;
5240         my $get_migration_ip = sub {
5241             my ($cidr, $nodename) = @_;
5242
5243             return $migration_ip if defined($migration_ip);
5244
5245             if (!defined($cidr)) {
5246                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5247                 $cidr = $dc_conf->{migration}->{network};
5248             }
5249
5250             if (defined($cidr)) {
5251                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5252
5253                 die "could not get IP: no address configured on local " .
5254                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5255
5256                 die "could not get IP: multiple addresses configured on local " .
5257                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5258
5259                 $migration_ip = @$ips[0];
5260             }
5261
5262             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5263                 if !defined($migration_ip);
5264
5265             return $migration_ip;
5266         };
5267
5268         my $migrate_uri;
5269         if ($statefile) {
5270             if ($statefile eq 'tcp') {
5271                 my $localip = "localhost";
5272                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5273                 my $nodename = nodename();
5274
5275                 if (!defined($migration_type)) {
5276                     if (defined($datacenterconf->{migration}->{type})) {
5277                         $migration_type = $datacenterconf->{migration}->{type};
5278                     } else {
5279                         $migration_type = 'secure';
5280                     }
5281                 }
5282
5283                 if ($migration_type eq 'insecure') {
5284                     $localip = $get_migration_ip->($migration_network, $nodename);
5285                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5286                 }
5287
5288                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5289                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5290                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5291                 push @$cmd, '-incoming', $migrate_uri;
5292                 push @$cmd, '-S';
5293
5294             } elsif ($statefile eq 'unix') {
5295                 # should be default for secure migrations as a ssh TCP forward
5296                 # tunnel is not deterministic reliable ready and fails regurarly
5297                 # to set up in time, so use UNIX socket forwards
5298                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5299                 unlink $socket_addr;
5300
5301                 $migrate_uri = "unix:$socket_addr";
5302
5303                 push @$cmd, '-incoming', $migrate_uri;
5304                 push @$cmd, '-S';
5305
5306             } elsif (-e $statefile) {
5307                 push @$cmd, '-loadstate', $statefile;
5308             } else {
5309                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5310                 push @$vollist, $statefile;
5311                 push @$cmd, '-loadstate', $statepath;
5312             }
5313         } elsif ($paused) {
5314             push @$cmd, '-S';
5315         }
5316
5317         # host pci devices
5318         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5319           my $d = parse_hostpci($conf->{"hostpci$i"});
5320           next if !$d;
5321           my $pcidevices = $d->{pciid};
5322           foreach my $pcidevice (@$pcidevices) {
5323                 my $pciid = $pcidevice->{id};
5324
5325                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5326                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5327                 die "no pci device info for device '$pciid'\n" if !$info;
5328
5329                 if ($d->{mdev}) {
5330                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5331                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5332                 } else {
5333                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5334                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5335                     die "can't reset pci device '$pciid'\n"
5336                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5337                 }
5338           }
5339         }
5340
5341         PVE::Storage::activate_volumes($storecfg, $vollist);
5342
5343         eval {
5344             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5345                 outfunc => sub {}, errfunc => sub {});
5346         };
5347         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5348         # timeout should be more than enough here...
5349         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5350
5351         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5352                                                   : $defaults->{cpuunits};
5353
5354         my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
5355         my %run_params = (
5356             timeout => $statefile ? undef : $start_timeout,
5357             umask => 0077,
5358             noerr => 1,
5359         );
5360
5361         # when migrating, prefix QEMU output so other side can pick up any
5362         # errors that might occur and show the user
5363         if ($migratedfrom) {
5364             $run_params{quiet} = 1;
5365             $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5366         }
5367
5368         my %properties = (
5369             Slice => 'qemu.slice',
5370             KillMode => 'none',
5371             CPUShares => $cpuunits
5372         );
5373
5374         if (my $cpulimit = $conf->{cpulimit}) {
5375             $properties{CPUQuota} = int($cpulimit * 100);
5376         }
5377         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5378
5379         my $run_qemu = sub {
5380             PVE::Tools::run_fork sub {
5381                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5382
5383                 my $exitcode = run_command($cmd, %run_params);
5384                 die "QEMU exited with code $exitcode\n" if $exitcode;
5385             };
5386         };
5387
5388         if ($conf->{hugepages}) {
5389
5390             my $code = sub {
5391                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5392                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5393
5394                 PVE::QemuServer::Memory::hugepages_mount();
5395                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5396
5397                 eval { $run_qemu->() };
5398                 if (my $err = $@) {
5399                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5400                     die $err;
5401                 }
5402
5403                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5404             };
5405             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5406
5407         } else {
5408             eval { $run_qemu->() };
5409         }
5410
5411         if (my $err = $@) {
5412             # deactivate volumes if start fails
5413             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5414             die "start failed: $err";
5415         }
5416
5417         print "migration listens on $migrate_uri\n" if $migrate_uri;
5418
5419         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5420             eval { mon_cmd($vmid, "cont"); };
5421             warn $@ if $@;
5422         }
5423
5424         #start nbd server for storage migration
5425         if ($targetstorage) {
5426             my $nodename = nodename();
5427             my $localip = $get_migration_ip->($migration_network, $nodename);
5428             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5429             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5430
5431             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5432
5433             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5434
5435             foreach my $opt (sort keys %$local_volumes) {
5436                 my $volid = $local_volumes->{$opt};
5437                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5438                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5439                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5440             }
5441         }
5442
5443         if ($migratedfrom) {
5444             eval {
5445                 set_migration_caps($vmid);
5446             };
5447             warn $@ if $@;
5448
5449             if ($spice_port) {
5450                 print "spice listens on port $spice_port\n";
5451                 if ($spice_ticket) {
5452                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5453                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5454                 }
5455             }
5456
5457         } else {
5458             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5459                 if !$statefile && $conf->{balloon};
5460
5461             foreach my $opt (keys %$conf) {
5462                 next if $opt !~  m/^net\d+$/;
5463                 my $nicconf = parse_net($conf->{$opt});
5464                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5465             }
5466         }
5467
5468         mon_cmd($vmid, 'qom-set',
5469                     path => "machine/peripheral/balloon0",
5470                     property => "guest-stats-polling-interval",
5471                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5472
5473         if ($is_suspended) {
5474             print "Resumed VM, removing state\n";
5475             if (my $vmstate = $conf->{vmstate}) {
5476                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5477                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5478             }
5479             delete $conf->@{qw(lock vmstate runningmachine)};
5480             PVE::QemuConfig->write_config($vmid, $conf);
5481         }
5482
5483         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5484     });
5485 }
5486
5487 sub vm_commandline {
5488     my ($storecfg, $vmid, $snapname) = @_;
5489
5490     my $conf = PVE::QemuConfig->load_config($vmid);
5491     my $forcemachine;
5492
5493     if ($snapname) {
5494         my $snapshot = $conf->{snapshots}->{$snapname};
5495         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5496
5497         # check for a 'runningmachine' in snapshot
5498         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5499
5500         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5501
5502         $conf = $snapshot;
5503     }
5504
5505     my $defaults = load_defaults();
5506
5507     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5508
5509     return PVE::Tools::cmd2string($cmd);
5510 }
5511
5512 sub vm_reset {
5513     my ($vmid, $skiplock) = @_;
5514
5515     PVE::QemuConfig->lock_config($vmid, sub {
5516
5517         my $conf = PVE::QemuConfig->load_config($vmid);
5518
5519         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5520
5521         mon_cmd($vmid, "system_reset");
5522     });
5523 }
5524
5525 sub get_vm_volumes {
5526     my ($conf) = @_;
5527
5528     my $vollist = [];
5529     foreach_volid($conf, sub {
5530         my ($volid, $attr) = @_;
5531
5532         return if $volid =~ m|^/|;
5533
5534         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5535         return if !$sid;
5536
5537         push @$vollist, $volid;
5538     });
5539
5540     return $vollist;
5541 }
5542
5543 sub vm_stop_cleanup {
5544     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5545
5546     eval {
5547
5548         if (!$keepActive) {
5549             my $vollist = get_vm_volumes($conf);
5550             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5551         }
5552
5553         foreach my $ext (qw(mon qmp pid vnc qga)) {
5554             unlink "/var/run/qemu-server/${vmid}.$ext";
5555         }
5556
5557         if ($conf->{ivshmem}) {
5558             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5559             # just delete it for now, VMs which have this already open do not
5560             # are affected, but new VMs will get a separated one. If this
5561             # becomes an issue we either add some sort of ref-counting or just
5562             # add a "don't delete on stop" flag to the ivshmem format.
5563             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5564         }
5565
5566         foreach my $key (keys %$conf) {
5567             next if $key !~ m/^hostpci(\d+)$/;
5568             my $hostpciindex = $1;
5569             my $d = parse_hostpci($conf->{$key});
5570             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5571
5572             foreach my $pci (@{$d->{pciid}}) {
5573                 my $pciid = $pci->{id};
5574                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5575             }
5576         }
5577
5578         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5579     };
5580     warn $@ if $@; # avoid errors - just warn
5581 }
5582
5583 # call only in locked context
5584 sub _do_vm_stop {
5585     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5586
5587     my $pid = check_running($vmid, $nocheck);
5588     return if !$pid;
5589
5590     my $conf;
5591     if (!$nocheck) {
5592         $conf = PVE::QemuConfig->load_config($vmid);
5593         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5594         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5595             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5596             $timeout = $opts->{down} if $opts->{down};
5597         }
5598         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5599     }
5600
5601     eval {
5602         if ($shutdown) {
5603             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5604                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5605             } else {
5606                 mon_cmd($vmid, "system_powerdown");
5607             }
5608         } else {
5609             mon_cmd($vmid, "quit");
5610         }
5611     };
5612     my $err = $@;
5613
5614     if (!$err) {
5615         $timeout = 60 if !defined($timeout);
5616
5617         my $count = 0;
5618         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5619             $count++;
5620             sleep 1;
5621         }
5622
5623         if ($count >= $timeout) {
5624             if ($force) {
5625                 warn "VM still running - terminating now with SIGTERM\n";
5626                 kill 15, $pid;
5627             } else {
5628                 die "VM quit/powerdown failed - got timeout\n";
5629             }
5630         } else {
5631             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5632             return;
5633         }
5634     } else {
5635         if ($force) {
5636             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5637             kill 15, $pid;
5638         } else {
5639             die "VM quit/powerdown failed\n";
5640         }
5641     }
5642
5643     # wait again
5644     $timeout = 10;
5645
5646     my $count = 0;
5647     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5648         $count++;
5649         sleep 1;
5650     }
5651
5652     if ($count >= $timeout) {
5653         warn "VM still running - terminating now with SIGKILL\n";
5654         kill 9, $pid;
5655         sleep 1;
5656     }
5657
5658     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5659 }
5660
5661 # Note: use $nocheck to skip tests if VM configuration file exists.
5662 # We need that when migration VMs to other nodes (files already moved)
5663 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5664 sub vm_stop {
5665     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5666
5667     $force = 1 if !defined($force) && !$shutdown;
5668
5669     if ($migratedfrom){
5670         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5671         kill 15, $pid if $pid;
5672         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5673         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5674         return;
5675     }
5676
5677     PVE::QemuConfig->lock_config($vmid, sub {
5678         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5679    });
5680 }
5681
5682 sub vm_reboot {
5683     my ($vmid, $timeout) = @_;
5684
5685     PVE::QemuConfig->lock_config($vmid, sub {
5686         eval {
5687
5688             # only reboot if running, as qmeventd starts it again on a stop event
5689             return if !check_running($vmid);
5690
5691             create_reboot_request($vmid);
5692
5693             my $storecfg = PVE::Storage::config();
5694             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5695
5696         };
5697         if (my $err = $@) {
5698             # avoid that the next normal shutdown will be confused for a reboot
5699             clear_reboot_request($vmid);
5700             die $err;
5701         }
5702    });
5703 }
5704
5705 # note: if using the statestorage parameter, the caller has to check privileges
5706 sub vm_suspend {
5707     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5708
5709     my $conf;
5710     my $path;
5711     my $storecfg;
5712     my $vmstate;
5713
5714     PVE::QemuConfig->lock_config($vmid, sub {
5715
5716         $conf = PVE::QemuConfig->load_config($vmid);
5717
5718         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5719         PVE::QemuConfig->check_lock($conf)
5720             if !($skiplock || $is_backing_up);
5721
5722         die "cannot suspend to disk during backup\n"
5723             if $is_backing_up && $includestate;
5724
5725         if ($includestate) {
5726             $conf->{lock} = 'suspending';
5727             my $date = strftime("%Y-%m-%d", localtime(time()));
5728             $storecfg = PVE::Storage::config();
5729             if (!$statestorage) {
5730                 $statestorage = find_vmstate_storage($conf, $storecfg);
5731                 # check permissions for the storage
5732                 my $rpcenv = PVE::RPCEnvironment::get();
5733                 if ($rpcenv->{type} ne 'cli') {
5734                     my $authuser = $rpcenv->get_user();
5735                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5736                 }
5737             }
5738
5739
5740             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5741             $path = PVE::Storage::path($storecfg, $vmstate);
5742             PVE::QemuConfig->write_config($vmid, $conf);
5743         } else {
5744             mon_cmd($vmid, "stop");
5745         }
5746     });
5747
5748     if ($includestate) {
5749         # save vm state
5750         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5751
5752         eval {
5753             mon_cmd($vmid, "savevm-start", statefile => $path);
5754             for(;;) {
5755                 my $state = mon_cmd($vmid, "query-savevm");
5756                 if (!$state->{status}) {
5757                     die "savevm not active\n";
5758                 } elsif ($state->{status} eq 'active') {
5759                     sleep(1);
5760                     next;
5761                 } elsif ($state->{status} eq 'completed') {
5762                     print "State saved, quitting\n";
5763                     last;
5764                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5765                     die "query-savevm failed with error '$state->{error}'\n"
5766                 } else {
5767                     die "query-savevm returned status '$state->{status}'\n";
5768                 }
5769             }
5770         };
5771         my $err = $@;
5772
5773         PVE::QemuConfig->lock_config($vmid, sub {
5774             $conf = PVE::QemuConfig->load_config($vmid);
5775             if ($err) {
5776                 # cleanup, but leave suspending lock, to indicate something went wrong
5777                 eval {
5778                     mon_cmd($vmid, "savevm-end");
5779                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5780                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5781                     delete $conf->@{qw(vmstate runningmachine)};
5782                     PVE::QemuConfig->write_config($vmid, $conf);
5783                 };
5784                 warn $@ if $@;
5785                 die $err;
5786             }
5787
5788             die "lock changed unexpectedly\n"
5789                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5790
5791             mon_cmd($vmid, "quit");
5792             $conf->{lock} = 'suspended';
5793             PVE::QemuConfig->write_config($vmid, $conf);
5794         });
5795     }
5796 }
5797
5798 sub vm_resume {
5799     my ($vmid, $skiplock, $nocheck) = @_;
5800
5801     PVE::QemuConfig->lock_config($vmid, sub {
5802         my $res = mon_cmd($vmid, 'query-status');
5803         my $resume_cmd = 'cont';
5804
5805         if ($res->{status} && $res->{status} eq 'suspended') {
5806             $resume_cmd = 'system_wakeup';
5807         }
5808
5809         if (!$nocheck) {
5810
5811             my $conf = PVE::QemuConfig->load_config($vmid);
5812
5813             PVE::QemuConfig->check_lock($conf)
5814                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5815         }
5816
5817         mon_cmd($vmid, $resume_cmd);
5818     });
5819 }
5820
5821 sub vm_sendkey {
5822     my ($vmid, $skiplock, $key) = @_;
5823
5824     PVE::QemuConfig->lock_config($vmid, sub {
5825
5826         my $conf = PVE::QemuConfig->load_config($vmid);
5827
5828         # there is no qmp command, so we use the human monitor command
5829         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5830         die $res if $res ne '';
5831     });
5832 }
5833
5834 # vzdump restore implementaion
5835
5836 sub tar_archive_read_firstfile {
5837     my $archive = shift;
5838
5839     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5840
5841     # try to detect archive type first
5842     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5843         die "unable to open file '$archive'\n";
5844     my $firstfile = <$fh>;
5845     kill 15, $pid;
5846     close $fh;
5847
5848     die "ERROR: archive contaions no data\n" if !$firstfile;
5849     chomp $firstfile;
5850
5851     return $firstfile;
5852 }
5853
5854 sub tar_restore_cleanup {
5855     my ($storecfg, $statfile) = @_;
5856
5857     print STDERR "starting cleanup\n";
5858
5859     if (my $fd = IO::File->new($statfile, "r")) {
5860         while (defined(my $line = <$fd>)) {
5861             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5862                 my $volid = $2;
5863                 eval {
5864                     if ($volid =~ m|^/|) {
5865                         unlink $volid || die 'unlink failed\n';
5866                     } else {
5867                         PVE::Storage::vdisk_free($storecfg, $volid);
5868                     }
5869                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5870                 };
5871                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5872             } else {
5873                 print STDERR "unable to parse line in statfile - $line";
5874             }
5875         }
5876         $fd->close();
5877     }
5878 }
5879
5880 sub restore_archive {
5881     my ($archive, $vmid, $user, $opts) = @_;
5882
5883     my $format = $opts->{format};
5884     my $comp;
5885
5886     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5887         $format = 'tar' if !$format;
5888         $comp = 'gzip';
5889     } elsif ($archive =~ m/\.tar$/) {
5890         $format = 'tar' if !$format;
5891     } elsif ($archive =~ m/.tar.lzo$/) {
5892         $format = 'tar' if !$format;
5893         $comp = 'lzop';
5894     } elsif ($archive =~ m/\.vma$/) {
5895         $format = 'vma' if !$format;
5896     } elsif ($archive =~ m/\.vma\.gz$/) {
5897         $format = 'vma' if !$format;
5898         $comp = 'gzip';
5899     } elsif ($archive =~ m/\.vma\.lzo$/) {
5900         $format = 'vma' if !$format;
5901         $comp = 'lzop';
5902     } else {
5903         $format = 'vma' if !$format; # default
5904     }
5905
5906     # try to detect archive format
5907     if ($format eq 'tar') {
5908         return restore_tar_archive($archive, $vmid, $user, $opts);
5909     } else {
5910         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5911     }
5912 }
5913
5914 sub restore_update_config_line {
5915     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5916
5917     return if $line =~ m/^\#qmdump\#/;
5918     return if $line =~ m/^\#vzdump\#/;
5919     return if $line =~ m/^lock:/;
5920     return if $line =~ m/^unused\d+:/;
5921     return if $line =~ m/^parent:/;
5922
5923     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5924     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5925         # try to convert old 1.X settings
5926         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5927         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5928             my ($model, $macaddr) = split(/\=/, $devconfig);
5929             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5930             my $net = {
5931                 model => $model,
5932                 bridge => "vmbr$ind",
5933                 macaddr => $macaddr,
5934             };
5935             my $netstr = print_net($net);
5936
5937             print $outfd "net$cookie->{netcount}: $netstr\n";
5938             $cookie->{netcount}++;
5939         }
5940     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5941         my ($id, $netstr) = ($1, $2);
5942         my $net = parse_net($netstr);
5943         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5944         $netstr = print_net($net);
5945         print $outfd "$id: $netstr\n";
5946     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5947         my $virtdev = $1;
5948         my $value = $3;
5949         my $di = parse_drive($virtdev, $value);
5950         if (defined($di->{backup}) && !$di->{backup}) {
5951             print $outfd "#$line";
5952         } elsif ($map->{$virtdev}) {
5953             delete $di->{format}; # format can change on restore
5954             $di->{file} = $map->{$virtdev};
5955             $value = print_drive($di);
5956             print $outfd "$virtdev: $value\n";
5957         } else {
5958             print $outfd $line;
5959         }
5960     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5961         my $vmgenid = $1;
5962         if ($vmgenid ne '0') {
5963             # always generate a new vmgenid if there was a valid one setup
5964             $vmgenid = generate_uuid();
5965         }
5966         print $outfd "vmgenid: $vmgenid\n";
5967     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5968         my ($uuid, $uuid_str);
5969         UUID::generate($uuid);
5970         UUID::unparse($uuid, $uuid_str);
5971         my $smbios1 = parse_smbios1($2);
5972         $smbios1->{uuid} = $uuid_str;
5973         print $outfd $1.print_smbios1($smbios1)."\n";
5974     } else {
5975         print $outfd $line;
5976     }
5977 }
5978
5979 sub scan_volids {
5980     my ($cfg, $vmid) = @_;
5981
5982     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5983
5984     my $volid_hash = {};
5985     foreach my $storeid (keys %$info) {
5986         foreach my $item (@{$info->{$storeid}}) {
5987             next if !($item->{volid} && $item->{size});
5988             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5989             $volid_hash->{$item->{volid}} = $item;
5990         }
5991     }
5992
5993     return $volid_hash;
5994 }
5995
5996 sub is_volume_in_use {
5997     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5998
5999     my $path = PVE::Storage::path($storecfg, $volid);
6000
6001     my $scan_config = sub {
6002         my ($cref, $snapname) = @_;
6003
6004         foreach my $key (keys %$cref) {
6005             my $value = $cref->{$key};
6006             if (is_valid_drivename($key)) {
6007                 next if $skip_drive && $key eq $skip_drive;
6008                 my $drive = parse_drive($key, $value);
6009                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6010                 return 1 if $volid eq $drive->{file};
6011                 if ($drive->{file} =~ m!^/!) {
6012                     return 1 if $drive->{file} eq $path;
6013                 } else {
6014                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6015                     next if !$storeid;
6016                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6017                     next if !$scfg;
6018                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6019                 }
6020             }
6021         }
6022
6023         return 0;
6024     };
6025
6026     return 1 if &$scan_config($conf);
6027
6028     undef $skip_drive;
6029
6030     foreach my $snapname (keys %{$conf->{snapshots}}) {
6031         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6032     }
6033
6034     return 0;
6035 }
6036
6037 sub update_disksize {
6038     my ($drive, $volid_hash) = @_;
6039
6040     my $volid = $drive->{file};
6041     return undef if !defined($volid);
6042
6043     my $oldsize = $drive->{size};
6044     my $newsize = $volid_hash->{$volid}->{size};
6045
6046     if (defined($newsize) && defined($oldsize) && $newsize != $oldsize) {
6047         $drive->{size} = $newsize;
6048
6049         my $old_fmt = PVE::JSONSchema::format_size($oldsize);
6050         my $new_fmt = PVE::JSONSchema::format_size($newsize);
6051
6052         return wantarray ? ($drive, $old_fmt, $new_fmt) : $drive;
6053     }
6054
6055     return undef;
6056 }
6057
6058 sub update_disk_config {
6059     my ($vmid, $conf, $volid_hash) = @_;
6060
6061     my $changes;
6062     my $prefix = "VM $vmid:";
6063
6064     # used and unused disks
6065     my $referenced = {};
6066
6067     # Note: it is allowed to define multiple storages with same path (alias), so
6068     # we need to check both 'volid' and real 'path' (two different volid can point
6069     # to the same path).
6070
6071     my $referencedpath = {};
6072
6073     # update size info
6074     foreach my $opt (keys %$conf) {
6075         if (is_valid_drivename($opt)) {
6076             my $drive = parse_drive($opt, $conf->{$opt});
6077             my $volid = $drive->{file};
6078             next if !$volid;
6079
6080             # mark volid as "in-use" for next step
6081             $referenced->{$volid} = 1;
6082             if ($volid_hash->{$volid} &&
6083                 (my $path = $volid_hash->{$volid}->{path})) {
6084                 $referencedpath->{$path} = 1;
6085             }
6086
6087             next if drive_is_cdrom($drive);
6088             next if !$volid_hash->{$volid};
6089
6090             my ($updated, $old_size, $new_size) = update_disksize($drive, $volid_hash);
6091             if (defined($updated)) {
6092                 $changes = 1;
6093                 $conf->{$opt} = print_drive($updated);
6094                 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
6095             }
6096         }
6097     }
6098
6099     # remove 'unusedX' entry if volume is used
6100     foreach my $opt (keys %$conf) {
6101         next if $opt !~ m/^unused\d+$/;
6102         my $volid = $conf->{$opt};
6103         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6104         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6105             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6106             $changes = 1;
6107             delete $conf->{$opt};
6108         }
6109
6110         $referenced->{$volid} = 1;
6111         $referencedpath->{$path} = 1 if $path;
6112     }
6113
6114     foreach my $volid (sort keys %$volid_hash) {
6115         next if $volid =~ m/vm-$vmid-state-/;
6116         next if $referenced->{$volid};
6117         my $path = $volid_hash->{$volid}->{path};
6118         next if !$path; # just to be sure
6119         next if $referencedpath->{$path};
6120         $changes = 1;
6121         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6122         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6123         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6124     }
6125
6126     return $changes;
6127 }
6128
6129 sub rescan {
6130     my ($vmid, $nolock, $dryrun) = @_;
6131
6132     my $cfg = PVE::Storage::config();
6133
6134     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6135     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6136     foreach my $stor (keys %{$cfg->{ids}}) {
6137         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6138     }
6139
6140     print "rescan volumes...\n";
6141     my $volid_hash = scan_volids($cfg, $vmid);
6142
6143     my $updatefn =  sub {
6144         my ($vmid) = @_;
6145
6146         my $conf = PVE::QemuConfig->load_config($vmid);
6147
6148         PVE::QemuConfig->check_lock($conf);
6149
6150         my $vm_volids = {};
6151         foreach my $volid (keys %$volid_hash) {
6152             my $info = $volid_hash->{$volid};
6153             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6154         }
6155
6156         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6157
6158         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6159     };
6160
6161     if (defined($vmid)) {
6162         if ($nolock) {
6163             &$updatefn($vmid);
6164         } else {
6165             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6166         }
6167     } else {
6168         my $vmlist = config_list();
6169         foreach my $vmid (keys %$vmlist) {
6170             if ($nolock) {
6171                 &$updatefn($vmid);
6172             } else {
6173                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6174             }
6175         }
6176     }
6177 }
6178
6179 sub restore_vma_archive {
6180     my ($archive, $vmid, $user, $opts, $comp) = @_;
6181
6182     my $readfrom = $archive;
6183
6184     my $cfg = PVE::Storage::config();
6185     my $commands = [];
6186     my $bwlimit = $opts->{bwlimit};
6187
6188     my $dbg_cmdstring = '';
6189     my $add_pipe = sub {
6190         my ($cmd) = @_;
6191         push @$commands, $cmd;
6192         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6193         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6194         $readfrom = '-';
6195     };
6196
6197     my $input = undef;
6198     if ($archive eq '-') {
6199         $input = '<&STDIN';
6200     } else {
6201         # If we use a backup from a PVE defined storage we also consider that
6202         # storage's rate limit:
6203         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6204         if (defined($volid)) {
6205             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6206             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6207             if ($readlimit) {
6208                 print STDERR "applying read rate limit: $readlimit\n";
6209                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6210                 $add_pipe->($cstream);
6211             }
6212         }
6213     }
6214
6215     if ($comp) {
6216         my $cmd;
6217         if ($comp eq 'gzip') {
6218             $cmd = ['zcat', $readfrom];
6219         } elsif ($comp eq 'lzop') {
6220             $cmd = ['lzop', '-d', '-c', $readfrom];
6221         } else {
6222             die "unknown compression method '$comp'\n";
6223         }
6224         $add_pipe->($cmd);
6225     }
6226
6227     my $tmpdir = "/var/tmp/vzdumptmp$$";
6228     rmtree $tmpdir;
6229
6230     # disable interrupts (always do cleanups)
6231     local $SIG{INT} =
6232         local $SIG{TERM} =
6233         local $SIG{QUIT} =
6234         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6235
6236     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6237     POSIX::mkfifo($mapfifo, 0600);
6238     my $fifofh;
6239
6240     my $openfifo = sub {
6241         open($fifofh, '>', $mapfifo) || die $!;
6242     };
6243
6244     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6245
6246     my $oldtimeout;
6247     my $timeout = 5;
6248
6249     my $devinfo = {};
6250
6251     my $rpcenv = PVE::RPCEnvironment::get();
6252
6253     my $conffile = PVE::QemuConfig->config_file($vmid);
6254     my $tmpfn = "$conffile.$$.tmp";
6255
6256     # Note: $oldconf is undef if VM does not exist
6257     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6258     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6259
6260     my %storage_limits;
6261
6262     my $print_devmap = sub {
6263         my $virtdev_hash = {};
6264
6265         my $cfgfn = "$tmpdir/qemu-server.conf";
6266
6267         # we can read the config - that is already extracted
6268         my $fh = IO::File->new($cfgfn, "r") ||
6269             "unable to read qemu-server.conf - $!\n";
6270
6271         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6272         if (-f $fwcfgfn) {
6273             my $pve_firewall_dir = '/etc/pve/firewall';
6274             mkdir $pve_firewall_dir; # make sure the dir exists
6275             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6276         }
6277
6278         while (defined(my $line = <$fh>)) {
6279             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6280                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6281                 die "archive does not contain data for drive '$virtdev'\n"
6282                     if !$devinfo->{$devname};
6283                 if (defined($opts->{storage})) {
6284                     $storeid = $opts->{storage} || 'local';
6285                 } elsif (!$storeid) {
6286                     $storeid = 'local';
6287                 }
6288                 $format = 'raw' if !$format;
6289                 $devinfo->{$devname}->{devname} = $devname;
6290                 $devinfo->{$devname}->{virtdev} = $virtdev;
6291                 $devinfo->{$devname}->{format} = $format;
6292                 $devinfo->{$devname}->{storeid} = $storeid;
6293
6294                 # check permission on storage
6295                 my $pool = $opts->{pool}; # todo: do we need that?
6296                 if ($user ne 'root@pam') {
6297                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6298                 }
6299
6300                 $storage_limits{$storeid} = $bwlimit;
6301
6302                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6303             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6304                 my $virtdev = $1;
6305                 my $drive = parse_drive($virtdev, $2);
6306                 if (drive_is_cloudinit($drive)) {
6307                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6308                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6309                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6310
6311                     my $d = {
6312                         format => $format,
6313                         storeid => $opts->{storage} // $storeid,
6314                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6315                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6316                         name => "vm-$vmid-cloudinit",
6317                         is_cloudinit => 1,
6318                     };
6319                     $virtdev_hash->{$virtdev} = $d;
6320                 }
6321             }
6322         }
6323
6324         foreach my $key (keys %storage_limits) {
6325             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6326             next if !$limit;
6327             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6328             $storage_limits{$key} = $limit * 1024;
6329         }
6330
6331         foreach my $devname (keys %$devinfo) {
6332             die "found no device mapping information for device '$devname'\n"
6333                 if !$devinfo->{$devname}->{virtdev};
6334         }
6335
6336         # create empty/temp config
6337         if ($oldconf) {
6338             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6339             foreach_drive($oldconf, sub {
6340                 my ($ds, $drive) = @_;
6341
6342                 return if drive_is_cdrom($drive, 1);
6343
6344                 my $volid = $drive->{file};
6345                 return if !$volid || $volid =~ m|^/|;
6346
6347                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6348                 return if !$path || !$owner || ($owner != $vmid);
6349
6350                 # Note: only delete disk we want to restore
6351                 # other volumes will become unused
6352                 if ($virtdev_hash->{$ds}) {
6353                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6354                     if (my $err = $@) {
6355                         warn $err;
6356                     }
6357                 }
6358             });
6359
6360             # delete vmstate files, after the restore we have no snapshots anymore
6361             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6362                 my $snap = $oldconf->{snapshots}->{$snapname};
6363                 if ($snap->{vmstate}) {
6364                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6365                     if (my $err = $@) {
6366                         warn $err;
6367                     }
6368                 }
6369             }
6370         }
6371
6372         my $map = {};
6373         foreach my $virtdev (sort keys %$virtdev_hash) {
6374             my $d = $virtdev_hash->{$virtdev};
6375             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6376             my $storeid = $d->{storeid};
6377             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6378
6379             my $map_opts = '';
6380             if (my $limit = $storage_limits{$storeid}) {
6381                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6382             }
6383
6384             # test if requested format is supported
6385             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6386             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6387             $d->{format} = $defFormat if !$supported;
6388
6389             my $name;
6390             if ($d->{is_cloudinit}) {
6391                 $name = $d->{name};
6392                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6393             }
6394
6395             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6396             print STDERR "new volume ID is '$volid'\n";
6397             $d->{volid} = $volid;
6398
6399             PVE::Storage::activate_volumes($cfg, [$volid]);
6400
6401             my $write_zeros = 1;
6402             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6403                 $write_zeros = 0;
6404             }
6405
6406             if (!$d->{is_cloudinit}) {
6407                 my $path = PVE::Storage::path($cfg, $volid);
6408
6409                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6410
6411                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6412             }
6413             $map->{$virtdev} = $volid;
6414         }
6415
6416         $fh->seek(0, 0) || die "seek failed - $!\n";
6417
6418         my $outfd = new IO::File ($tmpfn, "w") ||
6419             die "unable to write config for VM $vmid\n";
6420
6421         my $cookie = { netcount => 0 };
6422         while (defined(my $line = <$fh>)) {
6423             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6424         }
6425
6426         $fh->close();
6427         $outfd->close();
6428     };
6429
6430     eval {
6431         # enable interrupts
6432         local $SIG{INT} =
6433             local $SIG{TERM} =
6434             local $SIG{QUIT} =
6435             local $SIG{HUP} =
6436             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6437         local $SIG{ALRM} = sub { die "got timeout\n"; };
6438
6439         $oldtimeout = alarm($timeout);
6440
6441         my $parser = sub {
6442             my $line = shift;
6443
6444             print "$line\n";
6445
6446             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6447                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6448                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6449             } elsif ($line =~ m/^CTIME: /) {
6450                 # we correctly received the vma config, so we can disable
6451                 # the timeout now for disk allocation (set to 10 minutes, so
6452                 # that we always timeout if something goes wrong)
6453                 alarm(600);
6454                 &$print_devmap();
6455                 print $fifofh "done\n";
6456                 my $tmp = $oldtimeout || 0;
6457                 $oldtimeout = undef;
6458                 alarm($tmp);
6459                 close($fifofh);
6460             }
6461         };
6462
6463         print "restore vma archive: $dbg_cmdstring\n";
6464         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6465     };
6466     my $err = $@;
6467
6468     alarm($oldtimeout) if $oldtimeout;
6469
6470     my $vollist = [];
6471     foreach my $devname (keys %$devinfo) {
6472         my $volid = $devinfo->{$devname}->{volid};
6473         push @$vollist, $volid if $volid;
6474     }
6475
6476     PVE::Storage::deactivate_volumes($cfg, $vollist);
6477
6478     unlink $mapfifo;
6479
6480     if ($err) {
6481         rmtree $tmpdir;
6482         unlink $tmpfn;
6483
6484         foreach my $devname (keys %$devinfo) {
6485             my $volid = $devinfo->{$devname}->{volid};
6486             next if !$volid;
6487             eval {
6488                 if ($volid =~ m|^/|) {
6489                     unlink $volid || die 'unlink failed\n';
6490                 } else {
6491                     PVE::Storage::vdisk_free($cfg, $volid);
6492                 }
6493                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6494             };
6495             print STDERR "unable to cleanup '$volid' - $@" if $@;
6496         }
6497         die $err;
6498     }
6499
6500     rmtree $tmpdir;
6501
6502     rename($tmpfn, $conffile) ||
6503         die "unable to commit configuration file '$conffile'\n";
6504
6505     PVE::Cluster::cfs_update(); # make sure we read new file
6506
6507     eval { rescan($vmid, 1); };
6508     warn $@ if $@;
6509 }
6510
6511 sub restore_tar_archive {
6512     my ($archive, $vmid, $user, $opts) = @_;
6513
6514     if ($archive ne '-') {
6515         my $firstfile = tar_archive_read_firstfile($archive);
6516         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6517             if $firstfile ne 'qemu-server.conf';
6518     }
6519
6520     my $storecfg = PVE::Storage::config();
6521
6522     # avoid zombie disks when restoring over an existing VM -> cleanup first
6523     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6524     # skiplock=1 because qmrestore has set the 'create' lock itself already
6525     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6526     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6527
6528     my $tocmd = "/usr/lib/qemu-server/qmextract";
6529
6530     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6531     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6532     $tocmd .= ' --prealloc' if $opts->{prealloc};
6533     $tocmd .= ' --info' if $opts->{info};
6534
6535     # tar option "xf" does not autodetect compression when read from STDIN,
6536     # so we pipe to zcat
6537     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6538         PVE::Tools::shellquote("--to-command=$tocmd");
6539
6540     my $tmpdir = "/var/tmp/vzdumptmp$$";
6541     mkpath $tmpdir;
6542
6543     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6544     local $ENV{VZDUMP_VMID} = $vmid;
6545     local $ENV{VZDUMP_USER} = $user;
6546
6547     my $conffile = PVE::QemuConfig->config_file($vmid);
6548     my $tmpfn = "$conffile.$$.tmp";
6549
6550     # disable interrupts (always do cleanups)
6551     local $SIG{INT} =
6552         local $SIG{TERM} =
6553         local $SIG{QUIT} =
6554         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6555
6556     eval {
6557         # enable interrupts
6558         local $SIG{INT} =
6559             local $SIG{TERM} =
6560             local $SIG{QUIT} =
6561             local $SIG{HUP} =
6562             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6563
6564         if ($archive eq '-') {
6565             print "extracting archive from STDIN\n";
6566             run_command($cmd, input => "<&STDIN");
6567         } else {
6568             print "extracting archive '$archive'\n";
6569             run_command($cmd);
6570         }
6571
6572         return if $opts->{info};
6573
6574         # read new mapping
6575         my $map = {};
6576         my $statfile = "$tmpdir/qmrestore.stat";
6577         if (my $fd = IO::File->new($statfile, "r")) {
6578             while (defined (my $line = <$fd>)) {
6579                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6580                     $map->{$1} = $2 if $1;
6581                 } else {
6582                     print STDERR "unable to parse line in statfile - $line\n";
6583                 }
6584             }
6585             $fd->close();
6586         }
6587
6588         my $confsrc = "$tmpdir/qemu-server.conf";
6589
6590         my $srcfd = new IO::File($confsrc, "r") ||
6591             die "unable to open file '$confsrc'\n";
6592
6593         my $outfd = new IO::File ($tmpfn, "w") ||
6594             die "unable to write config for VM $vmid\n";
6595
6596         my $cookie = { netcount => 0 };
6597         while (defined (my $line = <$srcfd>)) {
6598             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6599         }
6600
6601         $srcfd->close();
6602         $outfd->close();
6603     };
6604     if (my $err = $@) {
6605         unlink $tmpfn;
6606         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6607         die $err;
6608     }
6609
6610     rmtree $tmpdir;
6611
6612     rename $tmpfn, $conffile ||
6613         die "unable to commit configuration file '$conffile'\n";
6614
6615     PVE::Cluster::cfs_update(); # make sure we read new file
6616
6617     eval { rescan($vmid, 1); };
6618     warn $@ if $@;
6619 };
6620
6621 sub foreach_storage_used_by_vm {
6622     my ($conf, $func) = @_;
6623
6624     my $sidhash = {};
6625
6626     foreach_drive($conf, sub {
6627         my ($ds, $drive) = @_;
6628         return if drive_is_cdrom($drive);
6629
6630         my $volid = $drive->{file};
6631
6632         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6633         $sidhash->{$sid} = $sid if $sid;
6634     });
6635
6636     foreach my $sid (sort keys %$sidhash) {
6637         &$func($sid);
6638     }
6639 }
6640
6641 my $qemu_snap_storage = {
6642     rbd => 1,
6643 };
6644 sub do_snapshots_with_qemu {
6645     my ($storecfg, $volid) = @_;
6646
6647     my $storage_name = PVE::Storage::parse_volume_id($volid);
6648     my $scfg = $storecfg->{ids}->{$storage_name};
6649
6650     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6651         return 1;
6652     }
6653
6654     if ($volid =~ m/\.(qcow2|qed)$/){
6655         return 1;
6656     }
6657
6658     return undef;
6659 }
6660
6661 sub qga_check_running {
6662     my ($vmid, $nowarn) = @_;
6663
6664     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6665     if ($@) {
6666         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6667         return 0;
6668     }
6669     return 1;
6670 }
6671
6672 sub template_create {
6673     my ($vmid, $conf, $disk) = @_;
6674
6675     my $storecfg = PVE::Storage::config();
6676
6677     foreach_drive($conf, sub {
6678         my ($ds, $drive) = @_;
6679
6680         return if drive_is_cdrom($drive);
6681         return if $disk && $ds ne $disk;
6682
6683         my $volid = $drive->{file};
6684         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6685
6686         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6687         $drive->{file} = $voliddst;
6688         $conf->{$ds} = print_drive($drive);
6689         PVE::QemuConfig->write_config($vmid, $conf);
6690     });
6691 }
6692
6693 sub convert_iscsi_path {
6694     my ($path) = @_;
6695
6696     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6697         my $portal = $1;
6698         my $target = $2;
6699         my $lun = $3;
6700
6701         my $initiator_name = get_initiator_name();
6702
6703         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6704                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6705     }
6706
6707     die "cannot convert iscsi path '$path', unkown format\n";
6708 }
6709
6710 sub qemu_img_convert {
6711     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6712
6713     my $storecfg = PVE::Storage::config();
6714     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6715     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6716
6717     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6718
6719     my $cachemode;
6720     my $src_path;
6721     my $src_is_iscsi = 0;
6722     my $src_format;
6723
6724     if ($src_storeid) {
6725         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6726         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6727         $src_format = qemu_img_format($src_scfg, $src_volname);
6728         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6729         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6730         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6731     } elsif (-f $src_volid) {
6732         $src_path = $src_volid;
6733         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6734             $src_format = $1;
6735         }
6736     }
6737
6738     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6739
6740     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6741     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6742     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6743     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6744
6745     my $cmd = [];
6746     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6747     push @$cmd, '-l', "snapshot.name=$snapname"
6748         if $snapname && $src_format && $src_format eq "qcow2";
6749     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6750     push @$cmd, '-T', $cachemode if defined($cachemode);
6751
6752     if ($src_is_iscsi) {
6753         push @$cmd, '--image-opts';
6754         $src_path = convert_iscsi_path($src_path);
6755     } elsif ($src_format) {
6756         push @$cmd, '-f', $src_format;
6757     }
6758
6759     if ($dst_is_iscsi) {
6760         push @$cmd, '--target-image-opts';
6761         $dst_path = convert_iscsi_path($dst_path);
6762     } else {
6763         push @$cmd, '-O', $dst_format;
6764     }
6765
6766     push @$cmd, $src_path;
6767
6768     if (!$dst_is_iscsi && $is_zero_initialized) {
6769         push @$cmd, "zeroinit:$dst_path";
6770     } else {
6771         push @$cmd, $dst_path;
6772     }
6773
6774     my $parser = sub {
6775         my $line = shift;
6776         if($line =~ m/\((\S+)\/100\%\)/){
6777             my $percent = $1;
6778             my $transferred = int($size * $percent / 100);
6779             my $remaining = $size - $transferred;
6780
6781             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6782         }
6783
6784     };
6785
6786     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6787     my $err = $@;
6788     die "copy failed: $err" if $err;
6789 }
6790
6791 sub qemu_img_format {
6792     my ($scfg, $volname) = @_;
6793
6794     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6795         return $1;
6796     } else {
6797         return "raw";
6798     }
6799 }
6800
6801 sub qemu_drive_mirror {
6802     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6803
6804     $jobs = {} if !$jobs;
6805
6806     my $qemu_target;
6807     my $format;
6808     $jobs->{"drive-$drive"} = {};
6809
6810     if ($dst_volid =~ /^nbd:/) {
6811         $qemu_target = $dst_volid;
6812         $format = "nbd";
6813     } else {
6814         my $storecfg = PVE::Storage::config();
6815         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6816
6817         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6818
6819         $format = qemu_img_format($dst_scfg, $dst_volname);
6820
6821         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6822
6823         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6824     }
6825
6826     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6827     $opts->{format} = $format if $format;
6828
6829     if (defined($bwlimit)) {
6830         $opts->{speed} = $bwlimit * 1024;
6831         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6832     } else {
6833         print "drive mirror is starting for drive-$drive\n";
6834     }
6835
6836     # if a job already runs for this device we get an error, catch it for cleanup
6837     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6838     if (my $err = $@) {
6839         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6840         warn "$@\n" if $@;
6841         die "mirroring error: $err\n";
6842     }
6843
6844     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6845 }
6846
6847 sub qemu_drive_mirror_monitor {
6848     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6849
6850     eval {
6851         my $err_complete = 0;
6852
6853         while (1) {
6854             die "storage migration timed out\n" if $err_complete > 300;
6855
6856             my $stats = mon_cmd($vmid, "query-block-jobs");
6857
6858             my $running_mirror_jobs = {};
6859             foreach my $stat (@$stats) {
6860                 next if $stat->{type} ne 'mirror';
6861                 $running_mirror_jobs->{$stat->{device}} = $stat;
6862             }
6863
6864             my $readycounter = 0;
6865
6866             foreach my $job (keys %$jobs) {
6867
6868                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6869                     print "$job : finished\n";
6870                     delete $jobs->{$job};
6871                     next;
6872                 }
6873
6874                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6875
6876                 my $busy = $running_mirror_jobs->{$job}->{busy};
6877                 my $ready = $running_mirror_jobs->{$job}->{ready};
6878                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6879                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6880                     my $remaining = $total - $transferred;
6881                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6882
6883                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6884                 }
6885
6886                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6887             }
6888
6889             last if scalar(keys %$jobs) == 0;
6890
6891             if ($readycounter == scalar(keys %$jobs)) {
6892                 print "all mirroring jobs are ready \n";
6893                 last if $skipcomplete; #do the complete later
6894
6895                 if ($vmiddst && $vmiddst != $vmid) {
6896                     my $agent_running = $qga && qga_check_running($vmid);
6897                     if ($agent_running) {
6898                         print "freeze filesystem\n";
6899                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6900                     } else {
6901                         print "suspend vm\n";
6902                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6903                     }
6904
6905                     # if we clone a disk for a new target vm, we don't switch the disk
6906                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6907
6908                     if ($agent_running) {
6909                         print "unfreeze filesystem\n";
6910                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6911                     } else {
6912                         print "resume vm\n";
6913                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6914                     }
6915
6916                     last;
6917                 } else {
6918
6919                     foreach my $job (keys %$jobs) {
6920                         # try to switch the disk if source and destination are on the same guest
6921                         print "$job: Completing block job...\n";
6922
6923                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6924                         if ($@ =~ m/cannot be completed/) {
6925                             print "$job: Block job cannot be completed, try again.\n";
6926                             $err_complete++;
6927                         }else {
6928                             print "$job: Completed successfully.\n";
6929                             $jobs->{$job}->{complete} = 1;
6930                         }
6931                     }
6932                 }
6933             }
6934             sleep 1;
6935         }
6936     };
6937     my $err = $@;
6938
6939     if ($err) {
6940         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6941         die "mirroring error: $err";
6942     }
6943
6944 }
6945
6946 sub qemu_blockjobs_cancel {
6947     my ($vmid, $jobs) = @_;
6948
6949     foreach my $job (keys %$jobs) {
6950         print "$job: Cancelling block job\n";
6951         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6952         $jobs->{$job}->{cancel} = 1;
6953     }
6954
6955     while (1) {
6956         my $stats = mon_cmd($vmid, "query-block-jobs");
6957
6958         my $running_jobs = {};
6959         foreach my $stat (@$stats) {
6960             $running_jobs->{$stat->{device}} = $stat;
6961         }
6962
6963         foreach my $job (keys %$jobs) {
6964
6965             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6966                 print "$job: Done.\n";
6967                 delete $jobs->{$job};
6968             }
6969         }
6970
6971         last if scalar(keys %$jobs) == 0;
6972
6973         sleep 1;
6974     }
6975 }
6976
6977 sub clone_disk {
6978     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6979         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6980
6981     my $newvolid;
6982
6983     if (!$full) {
6984         print "create linked clone of drive $drivename ($drive->{file})\n";
6985         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6986         push @$newvollist, $newvolid;
6987     } else {
6988
6989         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6990         $storeid = $storage if $storage;
6991
6992         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6993         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6994
6995         print "create full clone of drive $drivename ($drive->{file})\n";
6996         my $name = undef;
6997         if (drive_is_cloudinit($drive)) {
6998             $name = "vm-$newvmid-cloudinit";
6999             $name .= ".$dst_format" if $dst_format ne 'raw';
7000             $snapname = undef;
7001             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7002         }
7003         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7004         push @$newvollist, $newvolid;
7005
7006         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7007
7008         if (drive_is_cloudinit($drive)) {
7009             goto no_data_clone;
7010         }
7011
7012         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7013         if (!$running || $snapname) {
7014             # TODO: handle bwlimits
7015             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7016         } else {
7017
7018             my $kvmver = get_running_qemu_version ($vmid);
7019             if (!min_version($kvmver, 2, 7)) {
7020                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7021                     if $drive->{iothread};
7022             }
7023
7024             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7025         }
7026     }
7027
7028 no_data_clone:
7029     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7030
7031     my $disk = $drive;
7032     $disk->{format} = undef;
7033     $disk->{file} = $newvolid;
7034     $disk->{size} = $size;
7035
7036     return $disk;
7037 }
7038
7039 sub get_running_qemu_version {
7040     my ($vmid) = @_;
7041     my $res = mon_cmd($vmid, "query-version");
7042     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7043 }
7044
7045 sub qemu_use_old_bios_files {
7046     my ($machine_type) = @_;
7047
7048     return if !$machine_type;
7049
7050     my $use_old_bios_files = undef;
7051
7052     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7053         $machine_type = $1;
7054         $use_old_bios_files = 1;
7055     } else {
7056         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7057         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7058         # load new efi bios files on migration. So this hack is required to allow
7059         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7060         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7061         $use_old_bios_files = !min_version($version, 2, 4);
7062     }
7063
7064     return ($use_old_bios_files, $machine_type);
7065 }
7066
7067 sub create_efidisk($$$$$) {
7068     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7069
7070     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7071     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7072
7073     my $vars_size_b = -s $ovmf_vars;
7074     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7075     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7076     PVE::Storage::activate_volumes($storecfg, [$volid]);
7077
7078     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7079     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7080
7081     return ($volid, $size/1024);
7082 }
7083
7084 sub vm_iothreads_list {
7085     my ($vmid) = @_;
7086
7087     my $res = mon_cmd($vmid, 'query-iothreads');
7088
7089     my $iothreads = {};
7090     foreach my $iothread (@$res) {
7091         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7092     }
7093
7094     return $iothreads;
7095 }
7096
7097 sub scsihw_infos {
7098     my ($conf, $drive) = @_;
7099
7100     my $maxdev = 0;
7101
7102     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7103         $maxdev = 7;
7104     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7105         $maxdev = 1;
7106     } else {
7107         $maxdev = 256;
7108     }
7109
7110     my $controller = int($drive->{index} / $maxdev);
7111     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7112
7113     return ($maxdev, $controller, $controller_prefix);
7114 }
7115
7116 sub windows_version {
7117     my ($ostype) = @_;
7118
7119     return 0 if !$ostype;
7120
7121     my $winversion = 0;
7122
7123     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7124         $winversion = 5;
7125     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7126         $winversion = 6;
7127     } elsif ($ostype =~ m/^win(\d+)$/) {
7128         $winversion = $1;
7129     }
7130
7131     return $winversion;
7132 }
7133
7134 sub resolve_dst_disk_format {
7135         my ($storecfg, $storeid, $src_volname, $format) = @_;
7136         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7137
7138         if (!$format) {
7139             # if no target format is specified, use the source disk format as hint
7140             if ($src_volname) {
7141                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7142                 $format = qemu_img_format($scfg, $src_volname);
7143             } else {
7144                 return $defFormat;
7145             }
7146         }
7147
7148         # test if requested format is supported - else use default
7149         my $supported = grep { $_ eq $format } @$validFormats;
7150         $format = $defFormat if !$supported;
7151         return $format;
7152 }
7153
7154 sub resolve_first_disk {
7155     my $conf = shift;
7156     my @disks = PVE::QemuServer::valid_drive_names();
7157     my $firstdisk;
7158     foreach my $ds (reverse @disks) {
7159         next if !$conf->{$ds};
7160         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7161         next if PVE::QemuServer::drive_is_cdrom($disk);
7162         $firstdisk = $ds;
7163     }
7164     return $firstdisk;
7165 }
7166
7167 # NOTE: if this logic changes, please update docs & possibly gui logic
7168 sub find_vmstate_storage {
7169     my ($conf, $storecfg) = @_;
7170
7171     # first, return storage from conf if set
7172     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7173
7174     my ($target, $shared, $local);
7175
7176     foreach_storage_used_by_vm($conf, sub {
7177         my ($sid) = @_;
7178         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7179         my $dst = $scfg->{shared} ? \$shared : \$local;
7180         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7181     });
7182
7183     # second, use shared storage where VM has at least one disk
7184     # third, use local storage where VM has at least one disk
7185     # fall back to local storage
7186     $target = $shared // $local // 'local';
7187
7188     return $target;
7189 }
7190
7191 sub generate_uuid {
7192     my ($uuid, $uuid_str);
7193     UUID::generate($uuid);
7194     UUID::unparse($uuid, $uuid_str);
7195     return $uuid_str;
7196 }
7197
7198 sub generate_smbios1_uuid {
7199     return "uuid=".generate_uuid();
7200 }
7201
7202 sub nbd_stop {
7203     my ($vmid) = @_;
7204
7205     mon_cmd($vmid, 'nbd-server-stop');
7206 }
7207
7208 sub create_reboot_request {
7209     my ($vmid) = @_;
7210     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7211         or die "failed to create reboot trigger file: $!\n";
7212     close($fh);
7213 }
7214
7215 sub clear_reboot_request {
7216     my ($vmid) = @_;
7217     my $path = "/run/qemu-server/$vmid.reboot";
7218     my $res = 0;
7219
7220     $res = unlink($path);
7221     die "could not remove reboot request for $vmid: $!"
7222         if !$res && $! != POSIX::ENOENT;
7223
7224     return $res;
7225 }
7226
7227 # bash completion helper
7228
7229 sub complete_backup_archives {
7230     my ($cmdname, $pname, $cvalue) = @_;
7231
7232     my $cfg = PVE::Storage::config();
7233
7234     my $storeid;
7235
7236     if ($cvalue =~ m/^([^:]+):/) {
7237         $storeid = $1;
7238     }
7239
7240     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7241
7242     my $res = [];
7243     foreach my $id (keys %$data) {
7244         foreach my $item (@{$data->{$id}}) {
7245             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7246             push @$res, $item->{volid} if defined($item->{volid});
7247         }
7248     }
7249
7250     return $res;
7251 }
7252
7253 my $complete_vmid_full = sub {
7254     my ($running) = @_;
7255
7256     my $idlist = vmstatus();
7257
7258     my $res = [];
7259
7260     foreach my $id (keys %$idlist) {
7261         my $d = $idlist->{$id};
7262         if (defined($running)) {
7263             next if $d->{template};
7264             next if $running && $d->{status} ne 'running';
7265             next if !$running && $d->{status} eq 'running';
7266         }
7267         push @$res, $id;
7268
7269     }
7270     return $res;
7271 };
7272
7273 sub complete_vmid {
7274     return &$complete_vmid_full();
7275 }
7276
7277 sub complete_vmid_stopped {
7278     return &$complete_vmid_full(0);
7279 }
7280
7281 sub complete_vmid_running {
7282     return &$complete_vmid_full(1);
7283 }
7284
7285 sub complete_storage {
7286
7287     my $cfg = PVE::Storage::config();
7288     my $ids = $cfg->{ids};
7289
7290     my $res = [];
7291     foreach my $sid (keys %$ids) {
7292         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7293         next if !$ids->{$sid}->{content}->{images};
7294         push @$res, $sid;
7295     }
7296
7297     return $res;
7298 }
7299
7300 sub complete_migration_storage {
7301     my ($cmd, $param, $current_value, $all_args) = @_;
7302
7303     my $targetnode = @$all_args[1];
7304
7305     my $cfg = PVE::Storage::config();
7306     my $ids = $cfg->{ids};
7307
7308     my $res = [];
7309     foreach my $sid (keys %$ids) {
7310         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7311         next if !$ids->{$sid}->{content}->{images};
7312         push @$res, $sid;
7313     }
7314
7315     return $res;
7316 }
7317
7318 1;