PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Machine;
  48 use PVE::QemuServer::Memory;
  49 use PVE::QemuServer::Monitor qw(mon_cmd);
  50 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  51 use PVE::QemuServer::USB qw(parse_usb_device);
  52
  53 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  54 my $OVMF = {
  55     x86_64 => [
  56         "$EDK2_FW_BASE/OVMF_CODE.fd",
  57         "$EDK2_FW_BASE/OVMF_VARS.fd"
  58     ],
  59     aarch64 => [
  60         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  61         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  62     ],
  63 };
  64
  65 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  66
  67 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  68
  69 # Note about locking: we use flock on the config file protect
  70 # against concurent actions.
  71 # Aditionaly, we have a 'lock' setting in the config file. This
  72 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  73 # allowed when such lock is set. But you can ignore this kind of
  74 # lock with the --skiplock flag.
  75
  76 cfs_register_file('/qemu-server/',
  77                   \&parse_vm_config,
  78                   \&write_vm_config);
  79
  80 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  81     description => "Some command save/restore state from this location.",
  82     type => 'string',
  83     maxLength => 128,
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  88     type => 'string',
  89     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  90     description => "The drive's backing file's data format.",
  91     optional => 1,
  92 });
  93
  94 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  95         description => "Specifies the Qemu machine type.",
  96         type => 'string',
  97         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  98         maxLength => 40,
  99         optional => 1,
 100 });
 101
 102 #no warnings 'redefine';
 103
 104 sub cgroups_write {
 105    my ($controller, $vmid, $option, $value) = @_;
 106
 107    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 108    PVE::ProcFSTools::write_proc_entry($path, $value);
 109
 110 }
 111
 112 my $nodename_cache;
 113 sub nodename {
 114     $nodename_cache //= PVE::INotify::nodename();
 115     return $nodename_cache;
 116 }
 117
 118 my $watchdog_fmt = {
 119     model => {
 120         default_key => 1,
 121         type => 'string',
 122         enum => [qw(i6300esb ib700)],
 123         description => "Watchdog type to emulate.",
 124         default => 'i6300esb',
 125         optional => 1,
 126     },
 127     action => {
 128         type => 'string',
 129         enum => [qw(reset shutdown poweroff pause debug none)],
 130         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 131         optional => 1,
 132     },
 133 };
 134 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 135
 136 my $agent_fmt = {
 137     enabled => {
 138         description => "Enable/disable Qemu GuestAgent.",
 139         type => 'boolean',
 140         default => 0,
 141         default_key => 1,
 142     },
 143     fstrim_cloned_disks => {
 144         description => "Run fstrim after cloning/moving a disk.",
 145         type => 'boolean',
 146         optional => 1,
 147         default => 0
 148     },
 149     type => {
 150         description => "Select the agent type",
 151         type => 'string',
 152         default => 'virtio',
 153         optional => 1,
 154         enum => [qw(virtio isa)],
 155     },
 156 };
 157
 158 my $vga_fmt = {
 159     type => {
 160         description => "Select the VGA type.",
 161         type => 'string',
 162         default => 'std',
 163         optional => 1,
 164         default_key => 1,
 165         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 166     },
 167     memory => {
 168         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 169         type => 'integer',
 170         optional => 1,
 171         minimum => 4,
 172         maximum => 512,
 173     },
 174 };
 175
 176 my $ivshmem_fmt = {
 177     size => {
 178         type => 'integer',
 179         minimum => 1,
 180         description => "The size of the file in MB.",
 181     },
 182     name => {
 183         type => 'string',
 184         pattern => '[a-zA-Z0-9\-]+',
 185         optional => 1,
 186         format_description => 'string',
 187         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 188     },
 189 };
 190
 191 my $audio_fmt = {
 192     device => {
 193         type => 'string',
 194         enum => [qw(ich9-intel-hda intel-hda AC97)],
 195         description =>  "Configure an audio device."
 196     },
 197     driver =>  {
 198         type => 'string',
 199         enum => ['spice'],
 200         default => 'spice',
 201         optional => 1,
 202         description => "Driver backend for the audio device."
 203     },
 204 };
 205
 206 my $spice_enhancements_fmt = {
 207     foldersharing => {
 208         type => 'boolean',
 209         optional => 1,
 210         default => '0',
 211         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 212     },
 213     videostreaming =>  {
 214         type => 'string',
 215         enum => ['off', 'all', 'filter'],
 216         default => 'off',
 217         optional => 1,
 218         description => "Enable video streaming. Uses compression for detected video streams."
 219     },
 220 };
 221
 222 my $confdesc = {
 223     onboot => {
 224         optional => 1,
 225         type => 'boolean',
 226         description => "Specifies whether a VM will be started during system bootup.",
 227         default => 0,
 228     },
 229     autostart => {
 230         optional => 1,
 231         type => 'boolean',
 232         description => "Automatic restart after crash (currently ignored).",
 233         default => 0,
 234     },
 235     hotplug => {
 236         optional => 1,
 237         type => 'string', format => 'pve-hotplug-features',
 238         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 239         default => 'network,disk,usb',
 240     },
 241     reboot => {
 242         optional => 1,
 243         type => 'boolean',
 244         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 245         default => 1,
 246     },
 247     lock => {
 248         optional => 1,
 249         type => 'string',
 250         description => "Lock/unlock the VM.",
 251         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 252     },
 253     cpulimit => {
 254         optional => 1,
 255         type => 'number',
 256         description => "Limit of CPU usage.",
 257         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 258         minimum => 0,
 259         maximum => 128,
 260         default => 0,
 261     },
 262     cpuunits => {
 263         optional => 1,
 264         type => 'integer',
 265         description => "CPU weight for a VM.",
 266         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 267         minimum => 2,
 268         maximum => 262144,
 269         default => 1024,
 270     },
 271     memory => {
 272         optional => 1,
 273         type => 'integer',
 274         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 275         minimum => 16,
 276         default => 512,
 277     },
 278     balloon => {
 279         optional => 1,
 280         type => 'integer',
 281         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 282         minimum => 0,
 283     },
 284     shares => {
 285         optional => 1,
 286         type => 'integer',
 287         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 288         minimum => 0,
 289         maximum => 50000,
 290         default => 1000,
 291     },
 292     keyboard => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 296                        "It should not be necessary to set it.",
 297         enum => PVE::Tools::kvmkeymaplist(),
 298         default => undef,
 299     },
 300     name => {
 301         optional => 1,
 302         type => 'string', format => 'dns-name',
 303         description => "Set a name for the VM. Only used on the configuration web interface.",
 304     },
 305     scsihw => {
 306         optional => 1,
 307         type => 'string',
 308         description => "SCSI controller model",
 309         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 310         default => 'lsi',
 311     },
 312     description => {
 313         optional => 1,
 314         type => 'string',
 315         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 316     },
 317     ostype => {
 318         optional => 1,
 319         type => 'string',
 320         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 321         description => "Specify guest operating system.",
 322         verbose_description => <<EODESC,
 323 Specify guest operating system. This is used to enable special
 324 optimization/features for specific operating systems:
 325
 326 [horizontal]
 327 other;; unspecified OS
 328 wxp;; Microsoft Windows XP
 329 w2k;; Microsoft Windows 2000
 330 w2k3;; Microsoft Windows 2003
 331 w2k8;; Microsoft Windows 2008
 332 wvista;; Microsoft Windows Vista
 333 win7;; Microsoft Windows 7
 334 win8;; Microsoft Windows 8/2012/2012r2
 335 win10;; Microsoft Windows 10/2016
 336 l24;; Linux 2.4 Kernel
 337 l26;; Linux 2.6 - 5.X Kernel
 338 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 339 EODESC
 340     },
 341     boot => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 345         pattern => '[acdn]{1,4}',
 346         default => 'cdn',
 347     },
 348     bootdisk => {
 349         optional => 1,
 350         type => 'string', format => 'pve-qm-bootdisk',
 351         description => "Enable booting from specified disk.",
 352         pattern => '(ide|sata|scsi|virtio)\d+',
 353     },
 354     smp => {
 355         optional => 1,
 356         type => 'integer',
 357         description => "The number of CPUs. Please use option -sockets instead.",
 358         minimum => 1,
 359         default => 1,
 360     },
 361     sockets => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "The number of CPU sockets.",
 365         minimum => 1,
 366         default => 1,
 367     },
 368     cores => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "The number of cores per socket.",
 372         minimum => 1,
 373         default => 1,
 374     },
 375     numa => {
 376         optional => 1,
 377         type => 'boolean',
 378         description => "Enable/disable NUMA.",
 379         default => 0,
 380     },
 381     hugepages => {
 382         optional => 1,
 383         type => 'string',
 384         description => "Enable/disable hugepages memory.",
 385         enum => [qw(any 2 1024)],
 386     },
 387     vcpus => {
 388         optional => 1,
 389         type => 'integer',
 390         description => "Number of hotplugged vcpus.",
 391         minimum => 1,
 392         default => 0,
 393     },
 394     acpi => {
 395         optional => 1,
 396         type => 'boolean',
 397         description => "Enable/disable ACPI.",
 398         default => 1,
 399     },
 400     agent => {
 401         optional => 1,
 402         description => "Enable/disable Qemu GuestAgent and its properties.",
 403         type => 'string',
 404         format => $agent_fmt,
 405     },
 406     kvm => {
 407         optional => 1,
 408         type => 'boolean',
 409         description => "Enable/disable KVM hardware virtualization.",
 410         default => 1,
 411     },
 412     tdf => {
 413         optional => 1,
 414         type => 'boolean',
 415         description => "Enable/disable time drift fix.",
 416         default => 0,
 417     },
 418     localtime => {
 419         optional => 1,
 420         type => 'boolean',
 421         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 422     },
 423     freeze => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 427     },
 428     vga => {
 429         optional => 1,
 430         type => 'string', format => $vga_fmt,
 431         description => "Configure the VGA hardware.",
 432         verbose_description => "Configure the VGA Hardware. If you want to use ".
 433             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 434             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 435             "is 'std' for all OS types besides some Windows versions (XP and " .
 436             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 437             "display server. For win* OS you can select how many independent " .
 438             "displays you want, Linux guests can add displays them self.\n".
 439             "You can also run without any graphic card, using a serial device as terminal.",
 440     },
 441     watchdog => {
 442         optional => 1,
 443         type => 'string', format => 'pve-qm-watchdog',
 444         description => "Create a virtual hardware watchdog device.",
 445         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 446             " (by a guest action), the watchdog must be periodically polled " .
 447             "by an agent inside the guest or else the watchdog will reset " .
 448             "the guest (or execute the respective action specified)",
 449     },
 450     startdate => {
 451         optional => 1,
 452         type => 'string',
 453         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 454         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 455         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 456         default => 'now',
 457     },
 458     startup =>  get_standard_option('pve-startup-order'),
 459     template => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Enable/disable Template.",
 463         default => 0,
 464     },
 465     args => {
 466         optional => 1,
 467         type => 'string',
 468         description => "Arbitrary arguments passed to kvm.",
 469         verbose_description => <<EODESCR,
 470 Arbitrary arguments passed to kvm, for example:
 471
 472 args: -no-reboot -no-hpet
 473
 474 NOTE: this option is for experts only.
 475 EODESCR
 476     },
 477     tablet => {
 478         optional => 1,
 479         type => 'boolean',
 480         default => 1,
 481         description => "Enable/disable the USB tablet device.",
 482         verbose_description => "Enable/disable the USB tablet device. This device is " .
 483             "usually needed to allow absolute mouse positioning with VNC. " .
 484             "Else the mouse runs out of sync with normal VNC clients. " .
 485             "If you're running lots of console-only guests on one host, " .
 486             "you may consider disabling this to save some context switches. " .
 487             "This is turned off by default if you use spice (-vga=qxl).",
 488     },
 489     migrate_speed => {
 490         optional => 1,
 491         type => 'integer',
 492         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 493         minimum => 0,
 494         default => 0,
 495     },
 496     migrate_downtime => {
 497         optional => 1,
 498         type => 'number',
 499         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 500         minimum => 0,
 501         default => 0.1,
 502     },
 503     cdrom => {
 504         optional => 1,
 505         type => 'string', format => 'pve-qm-ide',
 506         typetext => '<volume>',
 507         description => "This is an alias for option -ide2",
 508     },
 509     cpu => {
 510         optional => 1,
 511         description => "Emulated CPU type.",
 512         type => 'string',
 513         format => $PVE::QemuServer::CPUConfig::cpu_fmt,
 514     },
 515     parent => get_standard_option('pve-snapshot-name', {
 516         optional => 1,
 517         description => "Parent snapshot name. This is used internally, and should not be modified.",
 518     }),
 519     snaptime => {
 520         optional => 1,
 521         description => "Timestamp for snapshots.",
 522         type => 'integer',
 523         minimum => 0,
 524     },
 525     vmstate => {
 526         optional => 1,
 527         type => 'string', format => 'pve-volume-id',
 528         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 529     },
 530     vmstatestorage => get_standard_option('pve-storage-id', {
 531         description => "Default storage for VM state volumes/files.",
 532         optional => 1,
 533     }),
 534     runningmachine => get_standard_option('pve-qemu-machine', {
 535         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 536     }),
 537     machine => get_standard_option('pve-qemu-machine'),
 538     arch => {
 539         description => "Virtual processor architecture. Defaults to the host.",
 540         optional => 1,
 541         type => 'string',
 542         enum => [qw(x86_64 aarch64)],
 543     },
 544     smbios1 => {
 545         description => "Specify SMBIOS type 1 fields.",
 546         type => 'string', format => 'pve-qm-smbios1',
 547         maxLength => 512,
 548         optional => 1,
 549     },
 550     protection => {
 551         optional => 1,
 552         type => 'boolean',
 553         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 554         default => 0,
 555     },
 556     bios => {
 557         optional => 1,
 558         type => 'string',
 559         enum => [ qw(seabios ovmf) ],
 560         description => "Select BIOS implementation.",
 561         default => 'seabios',
 562     },
 563     vmgenid => {
 564         type => 'string',
 565         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 566         format_description => 'UUID',
 567         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 568         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 569             " 128-bit integer value identifier to the guest OS. This allows to".
 570             " notify the guest operating system when the virtual machine is".
 571             " executed with a different configuration (e.g. snapshot execution".
 572             " or creation from a template). The guest operating system notices".
 573             " the change, and is then able to react as appropriate by marking".
 574             " its copies of distributed databases as dirty, re-initializing its".
 575             " random number generator, etc.\n".
 576             "Note that auto-creation only works when done throug API/CLI create".
 577             " or update methods, but not when manually editing the config file.",
 578         default => "1 (autogenerated)",
 579         optional => 1,
 580     },
 581     hookscript => {
 582         type => 'string',
 583         format => 'pve-volume-id',
 584         optional => 1,
 585         description => "Script that will be executed during various steps in the vms lifetime.",
 586     },
 587     ivshmem => {
 588         type => 'string',
 589         format => $ivshmem_fmt,
 590         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 591         optional => 1,
 592     },
 593     audio0 => {
 594         type => 'string',
 595         format => $audio_fmt,
 596         description => "Configure a audio device, useful in combination with QXL/Spice.",
 597         optional => 1
 598     },
 599     spice_enhancements => {
 600         type => 'string',
 601         format => $spice_enhancements_fmt,
 602         description => "Configure additional enhancements for SPICE.",
 603         optional => 1
 604     },
 605     tags => {
 606         type => 'string', format => 'pve-tag-list',
 607         description => 'Tags of the VM. This is only meta information.',
 608         optional => 1,
 609     },
 610 };
 611
 612 my $cicustom_fmt = {
 613     meta => {
 614         type => 'string',
 615         optional => 1,
 616         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 617         format => 'pve-volume-id',
 618         format_description => 'volume',
 619     },
 620     network => {
 621         type => 'string',
 622         optional => 1,
 623         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 624         format => 'pve-volume-id',
 625         format_description => 'volume',
 626     },
 627     user => {
 628         type => 'string',
 629         optional => 1,
 630         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 631         format => 'pve-volume-id',
 632         format_description => 'volume',
 633     },
 634 };
 635 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 636
 637 my $confdesc_cloudinit = {
 638     citype => {
 639         optional => 1,
 640         type => 'string',
 641         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 642         enum => ['configdrive2', 'nocloud'],
 643     },
 644     ciuser => {
 645         optional => 1,
 646         type => 'string',
 647         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 648     },
 649     cipassword => {
 650         optional => 1,
 651         type => 'string',
 652         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 653     },
 654     cicustom => {
 655         optional => 1,
 656         type => 'string',
 657         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 658         format => 'pve-qm-cicustom',
 659     },
 660     searchdomain => {
 661         optional => 1,
 662         type => 'string',
 663         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 664     },
 665     nameserver => {
 666         optional => 1,
 667         type => 'string', format => 'address-list',
 668         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 669     },
 670     sshkeys => {
 671         optional => 1,
 672         type => 'string',
 673         format => 'urlencoded',
 674         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 675     },
 676 };
 677
 678 # what about other qemu settings ?
 679 #cpu => 'string',
 680 #machine => 'string',
 681 #fda => 'file',
 682 #fdb => 'file',
 683 #mtdblock => 'file',
 684 #sd => 'file',
 685 #pflash => 'file',
 686 #snapshot => 'bool',
 687 #bootp => 'file',
 688 ##tftp => 'dir',
 689 ##smb => 'dir',
 690 #kernel => 'file',
 691 #append => 'string',
 692 #initrd => 'file',
 693 ##soundhw => 'string',
 694
 695 while (my ($k, $v) = each %$confdesc) {
 696     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 697 }
 698
 699 my $MAX_IDE_DISKS = 4;
 700 my $MAX_SCSI_DISKS = 31;
 701 my $MAX_VIRTIO_DISKS = 16;
 702 my $MAX_SATA_DISKS = 6;
 703 my $MAX_USB_DEVICES = 5;
 704 my $MAX_NETS = 32;
 705 my $MAX_UNUSED_DISKS = 256;
 706 my $MAX_HOSTPCI_DEVICES = 16;
 707 my $MAX_SERIAL_PORTS = 4;
 708 my $MAX_PARALLEL_PORTS = 3;
 709 my $MAX_NUMA = 8;
 710
 711 my $numa_fmt = {
 712     cpus => {
 713         type => "string",
 714         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 715         description => "CPUs accessing this NUMA node.",
 716         format_description => "id[-id];...",
 717     },
 718     memory => {
 719         type => "number",
 720         description => "Amount of memory this NUMA node provides.",
 721         optional => 1,
 722     },
 723     hostnodes => {
 724         type => "string",
 725         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 726         description => "Host NUMA nodes to use.",
 727         format_description => "id[-id];...",
 728         optional => 1,
 729     },
 730     policy => {
 731         type => 'string',
 732         enum => [qw(preferred bind interleave)],
 733         description => "NUMA allocation policy.",
 734         optional => 1,
 735     },
 736 };
 737 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 738 my $numadesc = {
 739     optional => 1,
 740     type => 'string', format => $numa_fmt,
 741     description => "NUMA topology.",
 742 };
 743 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 744
 745 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 746     $confdesc->{"numa$i"} = $numadesc;
 747 }
 748
 749 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 750                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 751                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 752 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 753
 754 my $net_fmt_bridge_descr = <<__EOD__;
 755 Bridge to attach the network device to. The Proxmox VE standard bridge
 756 is called 'vmbr0'.
 757
 758 If you do not specify a bridge, we create a kvm user (NATed) network
 759 device, which provides DHCP and DNS services. The following addresses
 760 are used:
 761
 762  10.0.2.2   Gateway
 763  10.0.2.3   DNS Server
 764  10.0.2.4   SMB Server
 765
 766 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 767 __EOD__
 768
 769 my $net_fmt = {
 770     macaddr  => get_standard_option('mac-addr', {
 771         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 772     }),
 773     model => {
 774         type => 'string',
 775         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 776         enum => $nic_model_list,
 777         default_key => 1,
 778     },
 779     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 780     bridge => {
 781         type => 'string',
 782         description => $net_fmt_bridge_descr,
 783         format_description => 'bridge',
 784         optional => 1,
 785     },
 786     queues => {
 787         type => 'integer',
 788         minimum => 0, maximum => 16,
 789         description => 'Number of packet queues to be used on the device.',
 790         optional => 1,
 791     },
 792     rate => {
 793         type => 'number',
 794         minimum => 0,
 795         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 796         optional => 1,
 797     },
 798     tag => {
 799         type => 'integer',
 800         minimum => 1, maximum => 4094,
 801         description => 'VLAN tag to apply to packets on this interface.',
 802         optional => 1,
 803     },
 804     trunks => {
 805         type => 'string',
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => 'VLAN trunks to pass through this interface.',
 808         format_description => 'vlanid[;vlanid...]',
 809         optional => 1,
 810     },
 811     firewall => {
 812         type => 'boolean',
 813         description => 'Whether this interface should be protected by the firewall.',
 814         optional => 1,
 815     },
 816     link_down => {
 817         type => 'boolean',
 818         description => 'Whether this interface should be disconnected (like pulling the plug).',
 819         optional => 1,
 820     },
 821 };
 822
 823 my $netdesc = {
 824     optional => 1,
 825     type => 'string', format => $net_fmt,
 826     description => "Specify network devices.",
 827 };
 828
 829 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 830
 831 my $ipconfig_fmt = {
 832     ip => {
 833         type => 'string',
 834         format => 'pve-ipv4-config',
 835         format_description => 'IPv4Format/CIDR',
 836         description => 'IPv4 address in CIDR format.',
 837         optional => 1,
 838         default => 'dhcp',
 839     },
 840     gw => {
 841         type => 'string',
 842         format => 'ipv4',
 843         format_description => 'GatewayIPv4',
 844         description => 'Default gateway for IPv4 traffic.',
 845         optional => 1,
 846         requires => 'ip',
 847     },
 848     ip6 => {
 849         type => 'string',
 850         format => 'pve-ipv6-config',
 851         format_description => 'IPv6Format/CIDR',
 852         description => 'IPv6 address in CIDR format.',
 853         optional => 1,
 854         default => 'dhcp',
 855     },
 856     gw6 => {
 857         type => 'string',
 858         format => 'ipv6',
 859         format_description => 'GatewayIPv6',
 860         description => 'Default gateway for IPv6 traffic.',
 861         optional => 1,
 862         requires => 'ip6',
 863     },
 864 };
 865 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 866 my $ipconfigdesc = {
 867     optional => 1,
 868     type => 'string', format => 'pve-qm-ipconfig',
 869     description => <<'EODESCR',
 870 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 871
 872 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 873
 874 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 875 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 876
 877 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 878 EODESCR
 879 };
 880 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 881
 882 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 883     $confdesc->{"net$i"} = $netdesc;
 884     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 885 }
 886
 887 foreach my $key (keys %$confdesc_cloudinit) {
 888     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 889 }
 890
 891 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 892 sub verify_volume_id_or_qm_path {
 893     my ($volid, $noerr) = @_;
 894
 895     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 896         return $volid;
 897     }
 898
 899     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 900     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 901     if ($@) {
 902         return undef if $noerr;
 903         die $@;
 904     }
 905     return $volid;
 906 }
 907
 908 my $drivename_hash;
 909
 910 my %drivedesc_base = (
 911     volume => { alias => 'file' },
 912     file => {
 913         type => 'string',
 914         format => 'pve-volume-id-or-qm-path',
 915         default_key => 1,
 916         format_description => 'volume',
 917         description => "The drive's backing volume.",
 918     },
 919     media => {
 920         type => 'string',
 921         enum => [qw(cdrom disk)],
 922         description => "The drive's media type.",
 923         default => 'disk',
 924         optional => 1
 925     },
 926     cyls => {
 927         type => 'integer',
 928         description => "Force the drive's physical geometry to have a specific cylinder count.",
 929         optional => 1
 930     },
 931     heads => {
 932         type => 'integer',
 933         description => "Force the drive's physical geometry to have a specific head count.",
 934         optional => 1
 935     },
 936     secs => {
 937         type => 'integer',
 938         description => "Force the drive's physical geometry to have a specific sector count.",
 939         optional => 1
 940     },
 941     trans => {
 942         type => 'string',
 943         enum => [qw(none lba auto)],
 944         description => "Force disk geometry bios translation mode.",
 945         optional => 1,
 946     },
 947     snapshot => {
 948         type => 'boolean',
 949         description => "Controls qemu's snapshot mode feature."
 950             . " If activated, changes made to the disk are temporary and will"
 951             . " be discarded when the VM is shutdown.",
 952         optional => 1,
 953     },
 954     cache => {
 955         type => 'string',
 956         enum => [qw(none writethrough writeback unsafe directsync)],
 957         description => "The drive's cache mode",
 958         optional => 1,
 959     },
 960     format => get_standard_option('pve-qm-image-format'),
 961     size => {
 962         type => 'string',
 963         format => 'disk-size',
 964         format_description => 'DiskSize',
 965         description => "Disk size. This is purely informational and has no effect.",
 966         optional => 1,
 967     },
 968     backup => {
 969         type => 'boolean',
 970         description => "Whether the drive should be included when making backups.",
 971         optional => 1,
 972     },
 973     replicate => {
 974         type => 'boolean',
 975         description => 'Whether the drive should considered for replication jobs.',
 976         optional => 1,
 977         default => 1,
 978     },
 979     rerror => {
 980         type => 'string',
 981         enum => [qw(ignore report stop)],
 982         description => 'Read error action.',
 983         optional => 1,
 984     },
 985     werror => {
 986         type => 'string',
 987         enum => [qw(enospc ignore report stop)],
 988         description => 'Write error action.',
 989         optional => 1,
 990     },
 991     aio => {
 992         type => 'string',
 993         enum => [qw(native threads)],
 994         description => 'AIO type to use.',
 995         optional => 1,
 996     },
 997     discard => {
 998         type => 'string',
 999         enum => [qw(ignore on)],
1000         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1001         optional => 1,
1002     },
1003     detect_zeroes => {
1004         type => 'boolean',
1005         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1006         optional => 1,
1007     },
1008     serial => {
1009         type => 'string',
1010         format => 'urlencoded',
1011         format_description => 'serial',
1012         maxLength => 20*3, # *3 since it's %xx url enoded
1013         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1014         optional => 1,
1015     },
1016     shared => {
1017         type => 'boolean',
1018         description => 'Mark this locally-managed volume as available on all nodes',
1019         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1020         optional => 1,
1021         default => 0,
1022     }
1023 );
1024
1025 my %iothread_fmt = ( iothread => {
1026         type => 'boolean',
1027         description => "Whether to use iothreads for this drive",
1028         optional => 1,
1029 });
1030
1031 my %model_fmt = (
1032     model => {
1033         type => 'string',
1034         format => 'urlencoded',
1035         format_description => 'model',
1036         maxLength => 40*3, # *3 since it's %xx url enoded
1037         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1038         optional => 1,
1039     },
1040 );
1041
1042 my %queues_fmt = (
1043     queues => {
1044         type => 'integer',
1045         description => "Number of queues.",
1046         minimum => 2,
1047         optional => 1
1048     }
1049 );
1050
1051 my %scsiblock_fmt = (
1052     scsiblock => {
1053         type => 'boolean',
1054         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1055         optional => 1,
1056         default => 0,
1057     },
1058 );
1059
1060 my %ssd_fmt = (
1061     ssd => {
1062         type => 'boolean',
1063         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1064         optional => 1,
1065     },
1066 );
1067
1068 my %wwn_fmt = (
1069     wwn => {
1070         type => 'string',
1071         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1072         format_description => 'wwn',
1073         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1074         optional => 1,
1075     },
1076 );
1077
1078 my $add_throttle_desc = sub {
1079     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1080     my $d = {
1081         type => $type,
1082         format_description => $unit,
1083         description => "Maximum $what in $longunit.",
1084         optional => 1,
1085     };
1086     $d->{minimum} = $minimum if defined($minimum);
1087     $drivedesc_base{$key} = $d;
1088 };
1089 # throughput: (leaky bucket)
1090 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1091 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1092 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1093 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1094 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1095 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1096 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1097 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1098 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1099
1100 # pools: (pool of IO before throttling starts taking effect)
1101 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1102 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1103 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1104 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1105 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1106 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1107
1108 # burst lengths
1109 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1110 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1111 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1112 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1113 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1114 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1115
1116 # legacy support
1117 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1118 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1119 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1120 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1121
1122 my $ide_fmt = {
1123     %drivedesc_base,
1124     %model_fmt,
1125     %ssd_fmt,
1126     %wwn_fmt,
1127 };
1128 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1129
1130 my $idedesc = {
1131     optional => 1,
1132     type => 'string', format => $ide_fmt,
1133     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1134 };
1135 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1136
1137 my $scsi_fmt = {
1138     %drivedesc_base,
1139     %iothread_fmt,
1140     %queues_fmt,
1141     %scsiblock_fmt,
1142     %ssd_fmt,
1143     %wwn_fmt,
1144 };
1145 my $scsidesc = {
1146     optional => 1,
1147     type => 'string', format => $scsi_fmt,
1148     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1149 };
1150 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1151
1152 my $sata_fmt = {
1153     %drivedesc_base,
1154     %ssd_fmt,
1155     %wwn_fmt,
1156 };
1157 my $satadesc = {
1158     optional => 1,
1159     type => 'string', format => $sata_fmt,
1160     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1161 };
1162 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1163
1164 my $virtio_fmt = {
1165     %drivedesc_base,
1166     %iothread_fmt,
1167 };
1168 my $virtiodesc = {
1169     optional => 1,
1170     type => 'string', format => $virtio_fmt,
1171     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1172 };
1173 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1174
1175 my $alldrive_fmt = {
1176     %drivedesc_base,
1177     %iothread_fmt,
1178     %model_fmt,
1179     %queues_fmt,
1180     %scsiblock_fmt,
1181     %ssd_fmt,
1182     %wwn_fmt,
1183 };
1184
1185 my $efidisk_fmt = {
1186     volume => { alias => 'file' },
1187     file => {
1188         type => 'string',
1189         format => 'pve-volume-id-or-qm-path',
1190         default_key => 1,
1191         format_description => 'volume',
1192         description => "The drive's backing volume.",
1193     },
1194     format => get_standard_option('pve-qm-image-format'),
1195     size => {
1196         type => 'string',
1197         format => 'disk-size',
1198         format_description => 'DiskSize',
1199         description => "Disk size. This is purely informational and has no effect.",
1200         optional => 1,
1201     },
1202 };
1203
1204 my $efidisk_desc = {
1205     optional => 1,
1206     type => 'string', format => $efidisk_fmt,
1207     description => "Configure a Disk for storing EFI vars",
1208 };
1209
1210 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1211
1212 my $usb_fmt = {
1213     host => {
1214         default_key => 1,
1215         type => 'string', format => 'pve-qm-usb-device',
1216         format_description => 'HOSTUSBDEVICE|spice',
1217         description => <<EODESCR,
1218 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1219
1220  'bus-port(.port)*' (decimal numbers) or
1221  'vendor_id:product_id' (hexadeciaml numbers) or
1222  'spice'
1223
1224 You can use the 'lsusb -t' command to list existing usb devices.
1225
1226 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1227
1228 The value 'spice' can be used to add a usb redirection devices for spice.
1229 EODESCR
1230     },
1231     usb3 => {
1232         optional => 1,
1233         type => 'boolean',
1234         description => "Specifies whether if given host option is a USB3 device or port.",
1235         default => 0,
1236     },
1237 };
1238
1239 my $usbdesc = {
1240     optional => 1,
1241     type => 'string', format => $usb_fmt,
1242     description => "Configure an USB device (n is 0 to 4).",
1243 };
1244 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1245
1246 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1247 my $hostpci_fmt = {
1248     host => {
1249         default_key => 1,
1250         type => 'string',
1251         pattern => qr/$PCIRE(;$PCIRE)*/,
1252         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1253         description => <<EODESCR,
1254 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1255 of PCI virtual functions of the host. HOSTPCIID syntax is:
1256
1257 'bus:dev.func' (hexadecimal numbers)
1258
1259 You can us the 'lspci' command to list existing PCI devices.
1260 EODESCR
1261     },
1262     rombar => {
1263         type => 'boolean',
1264         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1265         optional => 1,
1266         default => 1,
1267     },
1268     romfile => {
1269         type => 'string',
1270         pattern => '[^,;]+',
1271         format_description => 'string',
1272         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1273         optional => 1,
1274     },
1275     pcie => {
1276         type => 'boolean',
1277         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1278         optional => 1,
1279         default => 0,
1280     },
1281     'x-vga' => {
1282         type => 'boolean',
1283         description =>  "Enable vfio-vga device support.",
1284         optional => 1,
1285         default => 0,
1286     },
1287     'mdev' => {
1288         type => 'string',
1289         format_description => 'string',
1290         pattern => '[^/\.:]+',
1291         optional => 1,
1292         description => <<EODESCR
1293 The type of mediated device to use.
1294 An instance of this type will be created on startup of the VM and
1295 will be cleaned up when the VM stops.
1296 EODESCR
1297     }
1298 };
1299 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1300
1301 my $hostpcidesc = {
1302         optional => 1,
1303         type => 'string', format => 'pve-qm-hostpci',
1304         description => "Map host PCI devices into guest.",
1305         verbose_description =>  <<EODESCR,
1306 Map host PCI devices into guest.
1307
1308 NOTE: This option allows direct access to host hardware. So it is no longer
1309 possible to migrate such machines - use with special care.
1310
1311 CAUTION: Experimental! User reported problems with this option.
1312 EODESCR
1313 };
1314 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1315
1316 my $serialdesc = {
1317         optional => 1,
1318         type => 'string',
1319         pattern => '(/dev/.+|socket)',
1320         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1321         verbose_description =>  <<EODESCR,
1322 Create a serial device inside the VM (n is 0 to 3), and pass through a
1323 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1324 host side (use 'qm terminal' to open a terminal connection).
1325
1326 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1327
1328 CAUTION: Experimental! User reported problems with this option.
1329 EODESCR
1330 };
1331
1332 my $paralleldesc= {
1333         optional => 1,
1334         type => 'string',
1335         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1336         description =>  "Map host parallel devices (n is 0 to 2).",
1337         verbose_description =>  <<EODESCR,
1338 Map host parallel devices (n is 0 to 2).
1339
1340 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1341
1342 CAUTION: Experimental! User reported problems with this option.
1343 EODESCR
1344 };
1345
1346 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1347     $confdesc->{"parallel$i"} = $paralleldesc;
1348 }
1349
1350 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1351     $confdesc->{"serial$i"} = $serialdesc;
1352 }
1353
1354 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1355     $confdesc->{"hostpci$i"} = $hostpcidesc;
1356 }
1357
1358 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1359     $drivename_hash->{"ide$i"} = 1;
1360     $confdesc->{"ide$i"} = $idedesc;
1361 }
1362
1363 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1364     $drivename_hash->{"sata$i"} = 1;
1365     $confdesc->{"sata$i"} = $satadesc;
1366 }
1367
1368 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1369     $drivename_hash->{"scsi$i"} = 1;
1370     $confdesc->{"scsi$i"} = $scsidesc ;
1371 }
1372
1373 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1374     $drivename_hash->{"virtio$i"} = 1;
1375     $confdesc->{"virtio$i"} = $virtiodesc;
1376 }
1377
1378 $drivename_hash->{efidisk0} = 1;
1379 $confdesc->{efidisk0} = $efidisk_desc;
1380
1381 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1382     $confdesc->{"usb$i"} = $usbdesc;
1383 }
1384
1385 my $unuseddesc = {
1386     optional => 1,
1387     type => 'string', format => 'pve-volume-id',
1388     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1389 };
1390
1391 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1392     $confdesc->{"unused$i"} = $unuseddesc;
1393 }
1394
1395 my $kvm_api_version = 0;
1396
1397 sub kvm_version {
1398     return $kvm_api_version if $kvm_api_version;
1399
1400     open my $fh, '<', '/dev/kvm'
1401         or return undef;
1402
1403     # 0xae00 => KVM_GET_API_VERSION
1404     $kvm_api_version = ioctl($fh, 0xae00, 0);
1405
1406     return $kvm_api_version;
1407 }
1408
1409 my $kvm_user_version = {};
1410 my $kvm_mtime = {};
1411
1412 sub kvm_user_version {
1413     my ($binary) = @_;
1414
1415     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1416     my $st = stat($binary);
1417
1418     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1419     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1420         $cachedmtime == $st->mtime;
1421
1422     $kvm_user_version->{$binary} = 'unknown';
1423     $kvm_mtime->{$binary} = $st->mtime;
1424
1425     my $code = sub {
1426         my $line = shift;
1427         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1428             $kvm_user_version->{$binary} = $2;
1429         }
1430     };
1431
1432     eval { run_command([$binary, '--version'], outfunc => $code); };
1433     warn $@ if $@;
1434
1435     return $kvm_user_version->{$binary};
1436
1437 }
1438
1439 sub kernel_has_vhost_net {
1440     return -c '/dev/vhost-net';
1441 }
1442
1443 sub valid_drive_names {
1444     # order is important - used to autoselect boot disk
1445     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1446             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1447             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1448             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1449             'efidisk0');
1450 }
1451
1452 sub is_valid_drivename {
1453     my $dev = shift;
1454
1455     return defined($drivename_hash->{$dev});
1456 }
1457
1458 sub option_exists {
1459     my $key = shift;
1460     return defined($confdesc->{$key});
1461 }
1462
1463 my $cdrom_path;
1464 sub get_cdrom_path {
1465
1466     return  $cdrom_path if $cdrom_path;
1467
1468     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1469     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1470     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1471 }
1472
1473 sub get_iso_path {
1474     my ($storecfg, $vmid, $cdrom) = @_;
1475
1476     if ($cdrom eq 'cdrom') {
1477         return get_cdrom_path();
1478     } elsif ($cdrom eq 'none') {
1479         return '';
1480     } elsif ($cdrom =~ m|^/|) {
1481         return $cdrom;
1482     } else {
1483         return PVE::Storage::path($storecfg, $cdrom);
1484     }
1485 }
1486
1487 # try to convert old style file names to volume IDs
1488 sub filename_to_volume_id {
1489     my ($vmid, $file, $media) = @_;
1490
1491      if (!($file eq 'none' || $file eq 'cdrom' ||
1492           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1493
1494         return undef if $file =~ m|/|;
1495
1496         if ($media && $media eq 'cdrom') {
1497             $file = "local:iso/$file";
1498         } else {
1499             $file = "local:$vmid/$file";
1500         }
1501     }
1502
1503     return $file;
1504 }
1505
1506 sub verify_media_type {
1507     my ($opt, $vtype, $media) = @_;
1508
1509     return if !$media;
1510
1511     my $etype;
1512     if ($media eq 'disk') {
1513         $etype = 'images';
1514     } elsif ($media eq 'cdrom') {
1515         $etype = 'iso';
1516     } else {
1517         die "internal error";
1518     }
1519
1520     return if ($vtype eq $etype);
1521
1522     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1523 }
1524
1525 sub cleanup_drive_path {
1526     my ($opt, $storecfg, $drive) = @_;
1527
1528     # try to convert filesystem paths to volume IDs
1529
1530     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1531         ($drive->{file} !~ m|^/dev/.+|) &&
1532         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1533         ($drive->{file} !~ m/^\d+$/)) {
1534         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1535         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1536         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1537         verify_media_type($opt, $vtype, $drive->{media});
1538         $drive->{file} = $volid;
1539     }
1540
1541     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1542 }
1543
1544 sub parse_hotplug_features {
1545     my ($data) = @_;
1546
1547     my $res = {};
1548
1549     return $res if $data eq '0';
1550
1551     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1552
1553     foreach my $feature (PVE::Tools::split_list($data)) {
1554         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1555             $res->{$1} = 1;
1556         } else {
1557             die "invalid hotplug feature '$feature'\n";
1558         }
1559     }
1560     return $res;
1561 }
1562
1563 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1564 sub pve_verify_hotplug_features {
1565     my ($value, $noerr) = @_;
1566
1567     return $value if parse_hotplug_features($value);
1568
1569     return undef if $noerr;
1570
1571     die "unable to parse hotplug option\n";
1572 }
1573
1574 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1575 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1576 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1577 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1578 #        [,iothread=on][,serial=serial][,model=model]
1579
1580 sub parse_drive {
1581     my ($key, $data) = @_;
1582
1583     my ($interface, $index);
1584
1585     if ($key =~ m/^([^\d]+)(\d+)$/) {
1586         $interface = $1;
1587         $index = $2;
1588     } else {
1589         return undef;
1590     }
1591
1592     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1593                                      : $confdesc->{$key}->{format};
1594     if (!$desc) {
1595         warn "invalid drive key: $key\n";
1596         return undef;
1597     }
1598     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1599     return undef if !$res;
1600     $res->{interface} = $interface;
1601     $res->{index} = $index;
1602
1603     my $error = 0;
1604     foreach my $opt (qw(bps bps_rd bps_wr)) {
1605         if (my $bps = defined(delete $res->{$opt})) {
1606             if (defined($res->{"m$opt"})) {
1607                 warn "both $opt and m$opt specified\n";
1608                 ++$error;
1609                 next;
1610             }
1611             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1612         }
1613     }
1614
1615     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1616     for my $requirement (
1617         [mbps_max => 'mbps'],
1618         [mbps_rd_max => 'mbps_rd'],
1619         [mbps_wr_max => 'mbps_wr'],
1620         [miops_max => 'miops'],
1621         [miops_rd_max => 'miops_rd'],
1622         [miops_wr_max => 'miops_wr'],
1623         [bps_max_length => 'mbps_max'],
1624         [bps_rd_max_length => 'mbps_rd_max'],
1625         [bps_wr_max_length => 'mbps_wr_max'],
1626         [iops_max_length => 'iops_max'],
1627         [iops_rd_max_length => 'iops_rd_max'],
1628         [iops_wr_max_length => 'iops_wr_max']) {
1629         my ($option, $requires) = @$requirement;
1630         if ($res->{$option} && !$res->{$requires}) {
1631             warn "$option requires $requires\n";
1632             ++$error;
1633         }
1634     }
1635
1636     return undef if $error;
1637
1638     return undef if $res->{mbps_rd} && $res->{mbps};
1639     return undef if $res->{mbps_wr} && $res->{mbps};
1640     return undef if $res->{iops_rd} && $res->{iops};
1641     return undef if $res->{iops_wr} && $res->{iops};
1642
1643     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1644         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1645         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1646         return undef if $res->{interface} eq 'virtio';
1647     }
1648
1649     if (my $size = $res->{size}) {
1650         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1651     }
1652
1653     return $res;
1654 }
1655
1656 sub print_drive {
1657     my ($drive) = @_;
1658     my $data = { %$drive };
1659     delete $data->{$_} for qw(index interface);
1660     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1661 }
1662
1663 sub scsi_inquiry {
1664     my($fh, $noerr) = @_;
1665
1666     my $SG_IO = 0x2285;
1667     my $SG_GET_VERSION_NUM = 0x2282;
1668
1669     my $versionbuf = "\x00" x 8;
1670     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1671     if (!$ret) {
1672         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1673         return undef;
1674     }
1675     my $version = unpack("I", $versionbuf);
1676     if ($version < 30000) {
1677         die "scsi generic interface too old\n"  if !$noerr;
1678         return undef;
1679     }
1680
1681     my $buf = "\x00" x 36;
1682     my $sensebuf = "\x00" x 8;
1683     my $cmd = pack("C x3 C x1", 0x12, 36);
1684
1685     # see /usr/include/scsi/sg.h
1686     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1687
1688     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1689                       length($sensebuf), 0, length($buf), $buf,
1690                       $cmd, $sensebuf, 6000);
1691
1692     $ret = ioctl($fh, $SG_IO, $packet);
1693     if (!$ret) {
1694         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1695         return undef;
1696     }
1697
1698     my @res = unpack($sg_io_hdr_t, $packet);
1699     if ($res[17] || $res[18]) {
1700         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1701         return undef;
1702     }
1703
1704     my $res = {};
1705     (my $byte0, my $byte1, $res->{vendor},
1706      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1707
1708     $res->{removable} = $byte1 & 128 ? 1 : 0;
1709     $res->{type} = $byte0 & 31;
1710
1711     return $res;
1712 }
1713
1714 sub path_is_scsi {
1715     my ($path) = @_;
1716
1717     my $fh = IO::File->new("+<$path") || return undef;
1718     my $res = scsi_inquiry($fh, 1);
1719     close($fh);
1720
1721     return $res;
1722 }
1723
1724 sub print_tabletdevice_full {
1725     my ($conf, $arch) = @_;
1726
1727     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1728
1729     # we use uhci for old VMs because tablet driver was buggy in older qemu
1730     my $usbbus;
1731     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1732         $usbbus = 'ehci';
1733     } else {
1734         $usbbus = 'uhci';
1735     }
1736
1737     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1738 }
1739
1740 sub print_keyboarddevice_full {
1741     my ($conf, $arch, $machine) = @_;
1742
1743     return undef if $arch ne 'aarch64';
1744
1745     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1746 }
1747
1748 sub print_drivedevice_full {
1749     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1750
1751     my $device = '';
1752     my $maxdev = 0;
1753
1754     if ($drive->{interface} eq 'virtio') {
1755         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1756         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1757         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1758     } elsif ($drive->{interface} eq 'scsi') {
1759
1760         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1761         my $unit = $drive->{index} % $maxdev;
1762         my $devicetype = 'hd';
1763         my $path = '';
1764         if (drive_is_cdrom($drive)) {
1765             $devicetype = 'cd';
1766         } else {
1767             if ($drive->{file} =~ m|^/|) {
1768                 $path = $drive->{file};
1769                 if (my $info = path_is_scsi($path)) {
1770                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1771                         $devicetype = 'block';
1772                     } elsif ($info->{type} == 1) { # tape
1773                         $devicetype = 'generic';
1774                     }
1775                 }
1776             } else {
1777                  $path = PVE::Storage::path($storecfg, $drive->{file});
1778             }
1779
1780             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1781             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1782             if ($path =~ m/^iscsi\:\/\// &&
1783                !min_version($version, 4, 1)) {
1784                 $devicetype = 'generic';
1785             }
1786         }
1787
1788         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1789             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1790         } else {
1791             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1792         }
1793
1794         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1795             $device .= ",rotation_rate=1";
1796         }
1797         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1798
1799     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1800         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1801         my $controller = int($drive->{index} / $maxdev);
1802         my $unit = $drive->{index} % $maxdev;
1803         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1804
1805         $device = "ide-$devicetype";
1806         if ($drive->{interface} eq 'ide') {
1807             $device .= ",bus=ide.$controller,unit=$unit";
1808         } else {
1809             $device .= ",bus=ahci$controller.$unit";
1810         }
1811         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1812
1813         if ($devicetype eq 'hd') {
1814             if (my $model = $drive->{model}) {
1815                 $model = URI::Escape::uri_unescape($model);
1816                 $device .= ",model=$model";
1817             }
1818             if ($drive->{ssd}) {
1819                 $device .= ",rotation_rate=1";
1820             }
1821         }
1822         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1823     } elsif ($drive->{interface} eq 'usb') {
1824         die "implement me";
1825         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1826     } else {
1827         die "unsupported interface type";
1828     }
1829
1830     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1831
1832     if (my $serial = $drive->{serial}) {
1833         $serial = URI::Escape::uri_unescape($serial);
1834         $device .= ",serial=$serial";
1835     }
1836
1837
1838     return $device;
1839 }
1840
1841 sub get_initiator_name {
1842     my $initiator;
1843
1844     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1845     while (defined(my $line = <$fh>)) {
1846         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1847         $initiator = $1;
1848         last;
1849     }
1850     $fh->close();
1851
1852     return $initiator;
1853 }
1854
1855 sub print_drive_full {
1856     my ($storecfg, $vmid, $drive) = @_;
1857
1858     my $path;
1859     my $volid = $drive->{file};
1860     my $format;
1861
1862     if (drive_is_cdrom($drive)) {
1863         $path = get_iso_path($storecfg, $vmid, $volid);
1864     } else {
1865         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1866         if ($storeid) {
1867             $path = PVE::Storage::path($storecfg, $volid);
1868             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1869             $format = qemu_img_format($scfg, $volname);
1870         } else {
1871             $path = $volid;
1872             $format = "raw";
1873         }
1874    }
1875
1876     my $opts = '';
1877     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1878     foreach my $o (@qemu_drive_options) {
1879         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1880     }
1881
1882     # snapshot only accepts on|off
1883     if (defined($drive->{snapshot})) {
1884         my $v = $drive->{snapshot} ? 'on' : 'off';
1885         $opts .= ",snapshot=$v";
1886     }
1887
1888     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1889         my ($dir, $qmpname) = @$type;
1890         if (my $v = $drive->{"mbps$dir"}) {
1891             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1892         }
1893         if (my $v = $drive->{"mbps${dir}_max"}) {
1894             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1895         }
1896         if (my $v = $drive->{"bps${dir}_max_length"}) {
1897             $opts .= ",throttling.bps$qmpname-max-length=$v";
1898         }
1899         if (my $v = $drive->{"iops${dir}"}) {
1900             $opts .= ",throttling.iops$qmpname=$v";
1901         }
1902         if (my $v = $drive->{"iops${dir}_max"}) {
1903             $opts .= ",throttling.iops$qmpname-max=$v";
1904         }
1905         if (my $v = $drive->{"iops${dir}_max_length"}) {
1906             $opts .= ",throttling.iops$qmpname-max-length=$v";
1907         }
1908     }
1909
1910     $opts .= ",format=$format" if $format && !$drive->{format};
1911
1912     my $cache_direct = 0;
1913
1914     if (my $cache = $drive->{cache}) {
1915         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1916     } elsif (!drive_is_cdrom($drive)) {
1917         $opts .= ",cache=none";
1918         $cache_direct = 1;
1919     }
1920
1921     # aio native works only with O_DIRECT
1922     if (!$drive->{aio}) {
1923         if($cache_direct) {
1924             $opts .= ",aio=native";
1925         } else {
1926             $opts .= ",aio=threads";
1927         }
1928     }
1929
1930     if (!drive_is_cdrom($drive)) {
1931         my $detectzeroes;
1932         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1933             $detectzeroes = 'off';
1934         } elsif ($drive->{discard}) {
1935             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1936         } else {
1937             # This used to be our default with discard not being specified:
1938             $detectzeroes = 'on';
1939         }
1940         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1941     }
1942
1943     my $pathinfo = $path ? "file=$path," : '';
1944
1945     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1946 }
1947
1948 sub print_netdevice_full {
1949     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1950
1951     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1952
1953     my $device = $net->{model};
1954     if ($net->{model} eq 'virtio') {
1955          $device = 'virtio-net-pci';
1956      };
1957
1958     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1959     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1960     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1961         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1962         my $vectors = $net->{queues} * 2 + 2;
1963         $tmpstr .= ",vectors=$vectors,mq=on";
1964     }
1965     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1966
1967     if ($use_old_bios_files) {
1968         my $romfile;
1969         if ($device eq 'virtio-net-pci') {
1970             $romfile = 'pxe-virtio.rom';
1971         } elsif ($device eq 'e1000') {
1972             $romfile = 'pxe-e1000.rom';
1973         } elsif ($device eq 'ne2k') {
1974             $romfile = 'pxe-ne2k_pci.rom';
1975         } elsif ($device eq 'pcnet') {
1976             $romfile = 'pxe-pcnet.rom';
1977         } elsif ($device eq 'rtl8139') {
1978             $romfile = 'pxe-rtl8139.rom';
1979         }
1980         $tmpstr .= ",romfile=$romfile" if $romfile;
1981     }
1982
1983     return $tmpstr;
1984 }
1985
1986 sub print_netdev_full {
1987     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1988
1989     my $i = '';
1990     if ($netid =~ m/^net(\d+)$/) {
1991         $i = int($1);
1992     }
1993
1994     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1995
1996     my $ifname = "tap${vmid}i$i";
1997
1998     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1999     die "interface name '$ifname' is too long (max 15 character)\n"
2000         if length($ifname) >= 16;
2001
2002     my $vhostparam = '';
2003     if (is_native($arch)) {
2004         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2005     }
2006
2007     my $vmname = $conf->{name} || "vm$vmid";
2008
2009     my $netdev = "";
2010     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2011
2012     if ($net->{bridge}) {
2013         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2014     } else {
2015         $netdev = "type=user,id=$netid,hostname=$vmname";
2016     }
2017
2018     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2019
2020     return $netdev;
2021 }
2022
2023 my $vga_map = {
2024     'cirrus' => 'cirrus-vga',
2025     'std' => 'VGA',
2026     'vmware' => 'vmware-svga',
2027     'virtio' => 'virtio-vga',
2028 };
2029
2030 sub print_vga_device {
2031     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2032
2033     my $type = $vga_map->{$vga->{type}};
2034     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2035         $type = 'virtio-gpu';
2036     }
2037     my $vgamem_mb = $vga->{memory};
2038
2039     my $max_outputs = '';
2040     if ($qxlnum) {
2041         $type = $id ? 'qxl' : 'qxl-vga';
2042
2043         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2044             # set max outputs so linux can have up to 4 qxl displays with one device
2045             if (min_version($machine_version, 4, 1)) {
2046                 $max_outputs = ",max_outputs=4";
2047             }
2048         }
2049     }
2050
2051     die "no devicetype for $vga->{type}\n" if !$type;
2052
2053     my $memory = "";
2054     if ($vgamem_mb) {
2055         if ($vga->{type} eq 'virtio') {
2056             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2057             $memory = ",max_hostmem=$bytes";
2058         } elsif ($qxlnum) {
2059             # from https://www.spice-space.org/multiple-monitors.html
2060             $memory = ",vgamem_mb=$vga->{memory}";
2061             my $ram = $vgamem_mb * 4;
2062             my $vram = $vgamem_mb * 2;
2063             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2064         } else {
2065             $memory = ",vgamem_mb=$vga->{memory}";
2066         }
2067     } elsif ($qxlnum && $id) {
2068         $memory = ",ram_size=67108864,vram_size=33554432";
2069     }
2070
2071     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2072     my $vgaid = "vga" . ($id // '');
2073     my $pciaddr;
2074
2075     if ($q35 && $vgaid eq 'vga') {
2076         # the first display uses pcie.0 bus on q35 machines
2077         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2078     } else {
2079         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2080     }
2081
2082     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2083 }
2084
2085 sub drive_is_cloudinit {
2086     my ($drive) = @_;
2087     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2088 }
2089
2090 sub drive_is_cdrom {
2091     my ($drive, $exclude_cloudinit) = @_;
2092
2093     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2094
2095     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2096
2097 }
2098
2099 sub parse_number_sets {
2100     my ($set) = @_;
2101     my $res = [];
2102     foreach my $part (split(/;/, $set)) {
2103         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2104             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2105             push @$res, [ $1, $2 ];
2106         } else {
2107             die "invalid range: $part\n";
2108         }
2109     }
2110     return $res;
2111 }
2112
2113 sub parse_numa {
2114     my ($data) = @_;
2115
2116     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2117     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2118     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2119     return $res;
2120 }
2121
2122 sub parse_hostpci {
2123     my ($value) = @_;
2124
2125     return undef if !$value;
2126
2127     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2128
2129     my @idlist = split(/;/, $res->{host});
2130     delete $res->{host};
2131     foreach my $id (@idlist) {
2132         my $devs = PVE::SysFSTools::lspci($id);
2133         die "no PCI device found for '$id'\n" if !scalar(@$devs);
2134         push @{$res->{pciid}}, @$devs;
2135     }
2136     return $res;
2137 }
2138
2139 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2140 sub parse_net {
2141     my ($data) = @_;
2142
2143     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2144     if ($@) {
2145         warn $@;
2146         return undef;
2147     }
2148     if (!defined($res->{macaddr})) {
2149         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2150         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2151     }
2152     return $res;
2153 }
2154
2155 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2156 sub parse_ipconfig {
2157     my ($data) = @_;
2158
2159     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2160     if ($@) {
2161         warn $@;
2162         return undef;
2163     }
2164
2165     if ($res->{gw} && !$res->{ip}) {
2166         warn 'gateway specified without specifying an IP address';
2167         return undef;
2168     }
2169     if ($res->{gw6} && !$res->{ip6}) {
2170         warn 'IPv6 gateway specified without specifying an IPv6 address';
2171         return undef;
2172     }
2173     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2174         warn 'gateway specified together with DHCP';
2175         return undef;
2176     }
2177     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2178         # gw6 + auto/dhcp
2179         warn "IPv6 gateway specified together with $res->{ip6} address";
2180         return undef;
2181     }
2182
2183     if (!$res->{ip} && !$res->{ip6}) {
2184         return { ip => 'dhcp', ip6 => 'dhcp' };
2185     }
2186
2187     return $res;
2188 }
2189
2190 sub print_net {
2191     my $net = shift;
2192
2193     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2194 }
2195
2196 sub add_random_macs {
2197     my ($settings) = @_;
2198
2199     foreach my $opt (keys %$settings) {
2200         next if $opt !~ m/^net(\d+)$/;
2201         my $net = parse_net($settings->{$opt});
2202         next if !$net;
2203         $settings->{$opt} = print_net($net);
2204     }
2205 }
2206
2207 sub vm_is_volid_owner {
2208     my ($storecfg, $vmid, $volid) = @_;
2209
2210     if ($volid !~  m|^/|) {
2211         my ($path, $owner);
2212         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2213         if ($owner && ($owner == $vmid)) {
2214             return 1;
2215         }
2216     }
2217
2218     return undef;
2219 }
2220
2221 sub vmconfig_register_unused_drive {
2222     my ($storecfg, $vmid, $conf, $drive) = @_;
2223
2224     if (drive_is_cloudinit($drive)) {
2225         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2226         warn $@ if $@;
2227     } elsif (!drive_is_cdrom($drive)) {
2228         my $volid = $drive->{file};
2229         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2230             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2231         }
2232     }
2233 }
2234
2235 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2236 my $smbios1_fmt = {
2237     uuid => {
2238         type => 'string',
2239         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2240         format_description => 'UUID',
2241         description => "Set SMBIOS1 UUID.",
2242         optional => 1,
2243     },
2244     version => {
2245         type => 'string',
2246         pattern => '[A-Za-z0-9+\/]+={0,2}',
2247         format_description => 'Base64 encoded string',
2248         description => "Set SMBIOS1 version.",
2249         optional => 1,
2250     },
2251     serial => {
2252         type => 'string',
2253         pattern => '[A-Za-z0-9+\/]+={0,2}',
2254         format_description => 'Base64 encoded string',
2255         description => "Set SMBIOS1 serial number.",
2256         optional => 1,
2257     },
2258     manufacturer => {
2259         type => 'string',
2260         pattern => '[A-Za-z0-9+\/]+={0,2}',
2261         format_description => 'Base64 encoded string',
2262         description => "Set SMBIOS1 manufacturer.",
2263         optional => 1,
2264     },
2265     product => {
2266         type => 'string',
2267         pattern => '[A-Za-z0-9+\/]+={0,2}',
2268         format_description => 'Base64 encoded string',
2269         description => "Set SMBIOS1 product ID.",
2270         optional => 1,
2271     },
2272     sku => {
2273         type => 'string',
2274         pattern => '[A-Za-z0-9+\/]+={0,2}',
2275         format_description => 'Base64 encoded string',
2276         description => "Set SMBIOS1 SKU string.",
2277         optional => 1,
2278     },
2279     family => {
2280         type => 'string',
2281         pattern => '[A-Za-z0-9+\/]+={0,2}',
2282         format_description => 'Base64 encoded string',
2283         description => "Set SMBIOS1 family string.",
2284         optional => 1,
2285     },
2286     base64 => {
2287         type => 'boolean',
2288         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2289         optional => 1,
2290     },
2291 };
2292
2293 sub parse_smbios1 {
2294     my ($data) = @_;
2295
2296     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2297     warn $@ if $@;
2298     return $res;
2299 }
2300
2301 sub print_smbios1 {
2302     my ($smbios1) = @_;
2303     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2304 }
2305
2306 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2307
2308 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2309 sub verify_bootdisk {
2310     my ($value, $noerr) = @_;
2311
2312     return $value if is_valid_drivename($value);
2313
2314     return undef if $noerr;
2315
2316     die "invalid boot disk '$value'\n";
2317 }
2318
2319 sub parse_watchdog {
2320     my ($value) = @_;
2321
2322     return undef if !$value;
2323
2324     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2325     warn $@ if $@;
2326     return $res;
2327 }
2328
2329 sub parse_guest_agent {
2330     my ($value) = @_;
2331
2332     return {} if !defined($value->{agent});
2333
2334     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2335     warn $@ if $@;
2336
2337     # if the agent is disabled ignore the other potentially set properties
2338     return {} if !$res->{enabled};
2339     return $res;
2340 }
2341
2342 sub parse_vga {
2343     my ($value) = @_;
2344
2345     return {} if !$value;
2346     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2347     warn $@ if $@;
2348     return $res;
2349 }
2350
2351 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2352 sub verify_usb_device {
2353     my ($value, $noerr) = @_;
2354
2355     return $value if parse_usb_device($value);
2356
2357     return undef if $noerr;
2358
2359     die "unable to parse usb device\n";
2360 }
2361
2362 # add JSON properties for create and set function
2363 sub json_config_properties {
2364     my $prop = shift;
2365
2366     foreach my $opt (keys %$confdesc) {
2367         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2368         $prop->{$opt} = $confdesc->{$opt};
2369     }
2370
2371     return $prop;
2372 }
2373
2374 # return copy of $confdesc_cloudinit to generate documentation
2375 sub cloudinit_config_properties {
2376
2377     return dclone($confdesc_cloudinit);
2378 }
2379
2380 sub check_type {
2381     my ($key, $value) = @_;
2382
2383     die "unknown setting '$key'\n" if !$confdesc->{$key};
2384
2385     my $type = $confdesc->{$key}->{type};
2386
2387     if (!defined($value)) {
2388         die "got undefined value\n";
2389     }
2390
2391     if ($value =~ m/[\n\r]/) {
2392         die "property contains a line feed\n";
2393     }
2394
2395     if ($type eq 'boolean') {
2396         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2397         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2398         die "type check ('boolean') failed - got '$value'\n";
2399     } elsif ($type eq 'integer') {
2400         return int($1) if $value =~ m/^(\d+)$/;
2401         die "type check ('integer') failed - got '$value'\n";
2402     } elsif ($type eq 'number') {
2403         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2404         die "type check ('number') failed - got '$value'\n";
2405     } elsif ($type eq 'string') {
2406         if (my $fmt = $confdesc->{$key}->{format}) {
2407             PVE::JSONSchema::check_format($fmt, $value);
2408             return $value;
2409         }
2410         $value =~ s/^\"(.*)\"$/$1/;
2411         return $value;
2412     } else {
2413         die "internal error"
2414     }
2415 }
2416
2417 sub destroy_vm {
2418     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2419
2420     my $conf = PVE::QemuConfig->load_config($vmid);
2421
2422     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2423
2424     if ($conf->{template}) {
2425         # check if any base image is still used by a linked clone
2426         foreach_drive($conf, sub {
2427                 my ($ds, $drive) = @_;
2428                 return if drive_is_cdrom($drive);
2429
2430                 my $volid = $drive->{file};
2431                 return if !$volid || $volid =~ m|^/|;
2432
2433                 die "base volume '$volid' is still in use by linked cloned\n"
2434                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2435
2436         });
2437     }
2438
2439     # only remove disks owned by this VM
2440     foreach_drive($conf, sub {
2441         my ($ds, $drive) = @_;
2442         return if drive_is_cdrom($drive, 1);
2443
2444         my $volid = $drive->{file};
2445         return if !$volid || $volid =~ m|^/|;
2446
2447         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2448         return if !$path || !$owner || ($owner != $vmid);
2449
2450         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2451         warn "Could not remove disk '$volid', check manually: $@" if $@;
2452     });
2453
2454     # also remove unused disk
2455     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2456     PVE::Storage::foreach_volid($vmdisks, sub {
2457         my ($volid, $sid, $volname, $d) = @_;
2458         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2459         warn $@ if $@;
2460     });
2461
2462     if (defined $replacement_conf) {
2463         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2464     } else {
2465         PVE::QemuConfig->destroy_config($vmid);
2466     }
2467 }
2468
2469 sub parse_vm_config {
2470     my ($filename, $raw) = @_;
2471
2472     return undef if !defined($raw);
2473
2474     my $res = {
2475         digest => Digest::SHA::sha1_hex($raw),
2476         snapshots => {},
2477         pending => {},
2478     };
2479
2480     $filename =~ m|/qemu-server/(\d+)\.conf$|
2481         || die "got strange filename '$filename'";
2482
2483     my $vmid = $1;
2484
2485     my $conf = $res;
2486     my $descr;
2487     my $section = '';
2488
2489     my @lines = split(/\n/, $raw);
2490     foreach my $line (@lines) {
2491         next if $line =~ m/^\s*$/;
2492
2493         if ($line =~ m/^\[PENDING\]\s*$/i) {
2494             $section = 'pending';
2495             if (defined($descr)) {
2496                 $descr =~ s/\s+$//;
2497                 $conf->{description} = $descr;
2498             }
2499             $descr = undef;
2500             $conf = $res->{$section} = {};
2501             next;
2502
2503         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2504             $section = $1;
2505             if (defined($descr)) {
2506                 $descr =~ s/\s+$//;
2507                 $conf->{description} = $descr;
2508             }
2509             $descr = undef;
2510             $conf = $res->{snapshots}->{$section} = {};
2511             next;
2512         }
2513
2514         if ($line =~ m/^\#(.*)\s*$/) {
2515             $descr = '' if !defined($descr);
2516             $descr .= PVE::Tools::decode_text($1) . "\n";
2517             next;
2518         }
2519
2520         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2521             $descr = '' if !defined($descr);
2522             $descr .= PVE::Tools::decode_text($2);
2523         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2524             $conf->{snapstate} = $1;
2525         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2526             my $key = $1;
2527             my $value = $2;
2528             $conf->{$key} = $value;
2529         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2530             my $value = $1;
2531             if ($section eq 'pending') {
2532                 $conf->{delete} = $value; # we parse this later
2533             } else {
2534                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2535             }
2536         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2537             my $key = $1;
2538             my $value = $2;
2539             eval { $value = check_type($key, $value); };
2540             if ($@) {
2541                 warn "vm $vmid - unable to parse value of '$key' - $@";
2542             } else {
2543                 $key = 'ide2' if $key eq 'cdrom';
2544                 my $fmt = $confdesc->{$key}->{format};
2545                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2546                     my $v = parse_drive($key, $value);
2547                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2548                         $v->{file} = $volid;
2549                         $value = print_drive($v);
2550                     } else {
2551                         warn "vm $vmid - unable to parse value of '$key'\n";
2552                         next;
2553                     }
2554                 }
2555
2556                 $conf->{$key} = $value;
2557             }
2558         }
2559     }
2560
2561     if (defined($descr)) {
2562         $descr =~ s/\s+$//;
2563         $conf->{description} = $descr;
2564     }
2565     delete $res->{snapstate}; # just to be sure
2566
2567     return $res;
2568 }
2569
2570 sub write_vm_config {
2571     my ($filename, $conf) = @_;
2572
2573     delete $conf->{snapstate}; # just to be sure
2574
2575     if ($conf->{cdrom}) {
2576         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2577         $conf->{ide2} = $conf->{cdrom};
2578         delete $conf->{cdrom};
2579     }
2580
2581     # we do not use 'smp' any longer
2582     if ($conf->{sockets}) {
2583         delete $conf->{smp};
2584     } elsif ($conf->{smp}) {
2585         $conf->{sockets} = $conf->{smp};
2586         delete $conf->{cores};
2587         delete $conf->{smp};
2588     }
2589
2590     my $used_volids = {};
2591
2592     my $cleanup_config = sub {
2593         my ($cref, $pending, $snapname) = @_;
2594
2595         foreach my $key (keys %$cref) {
2596             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2597                 $key eq 'snapstate' || $key eq 'pending';
2598             my $value = $cref->{$key};
2599             if ($key eq 'delete') {
2600                 die "propertry 'delete' is only allowed in [PENDING]\n"
2601                     if !$pending;
2602                 # fixme: check syntax?
2603                 next;
2604             }
2605             eval { $value = check_type($key, $value); };
2606             die "unable to parse value of '$key' - $@" if $@;
2607
2608             $cref->{$key} = $value;
2609
2610             if (!$snapname && is_valid_drivename($key)) {
2611                 my $drive = parse_drive($key, $value);
2612                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2613             }
2614         }
2615     };
2616
2617     &$cleanup_config($conf);
2618
2619     &$cleanup_config($conf->{pending}, 1);
2620
2621     foreach my $snapname (keys %{$conf->{snapshots}}) {
2622         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2623         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2624     }
2625
2626     # remove 'unusedX' settings if we re-add a volume
2627     foreach my $key (keys %$conf) {
2628         my $value = $conf->{$key};
2629         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2630             delete $conf->{$key};
2631         }
2632     }
2633
2634     my $generate_raw_config = sub {
2635         my ($conf, $pending) = @_;
2636
2637         my $raw = '';
2638
2639         # add description as comment to top of file
2640         if (defined(my $descr = $conf->{description})) {
2641             if ($descr) {
2642                 foreach my $cl (split(/\n/, $descr)) {
2643                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2644                 }
2645             } else {
2646                 $raw .= "#\n" if $pending;
2647             }
2648         }
2649
2650         foreach my $key (sort keys %$conf) {
2651             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2652             $raw .= "$key: $conf->{$key}\n";
2653         }
2654         return $raw;
2655     };
2656
2657     my $raw = &$generate_raw_config($conf);
2658
2659     if (scalar(keys %{$conf->{pending}})){
2660         $raw .= "\n[PENDING]\n";
2661         $raw .= &$generate_raw_config($conf->{pending}, 1);
2662     }
2663
2664     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2665         $raw .= "\n[$snapname]\n";
2666         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2667     }
2668
2669     return $raw;
2670 }
2671
2672 sub load_defaults {
2673
2674     my $res = {};
2675
2676     # we use static defaults from our JSON schema configuration
2677     foreach my $key (keys %$confdesc) {
2678         if (defined(my $default = $confdesc->{$key}->{default})) {
2679             $res->{$key} = $default;
2680         }
2681     }
2682
2683     return $res;
2684 }
2685
2686 sub config_list {
2687     my $vmlist = PVE::Cluster::get_vmlist();
2688     my $res = {};
2689     return $res if !$vmlist || !$vmlist->{ids};
2690     my $ids = $vmlist->{ids};
2691     my $nodename = nodename();
2692
2693     foreach my $vmid (keys %$ids) {
2694         my $d = $ids->{$vmid};
2695         next if !$d->{node} || $d->{node} ne $nodename;
2696         next if !$d->{type} || $d->{type} ne 'qemu';
2697         $res->{$vmid}->{exists} = 1;
2698     }
2699     return $res;
2700 }
2701
2702 # test if VM uses local resources (to prevent migration)
2703 sub check_local_resources {
2704     my ($conf, $noerr) = @_;
2705
2706     my @loc_res = ();
2707
2708     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2709     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2710
2711     push @loc_res, "ivshmem" if $conf->{ivshmem};
2712
2713     foreach my $k (keys %$conf) {
2714         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2715         # sockets are safe: they will recreated be on the target side post-migrate
2716         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2717         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2718     }
2719
2720     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2721
2722     return \@loc_res;
2723 }
2724
2725 # check if used storages are available on all nodes (use by migrate)
2726 sub check_storage_availability {
2727     my ($storecfg, $conf, $node) = @_;
2728
2729     foreach_drive($conf, sub {
2730         my ($ds, $drive) = @_;
2731
2732         my $volid = $drive->{file};
2733         return if !$volid;
2734
2735         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2736         return if !$sid;
2737
2738         # check if storage is available on both nodes
2739         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2740         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2741    });
2742 }
2743
2744 # list nodes where all VM images are available (used by has_feature API)
2745 sub shared_nodes {
2746     my ($conf, $storecfg) = @_;
2747
2748     my $nodelist = PVE::Cluster::get_nodelist();
2749     my $nodehash = { map { $_ => 1 } @$nodelist };
2750     my $nodename = nodename();
2751
2752     foreach_drive($conf, sub {
2753         my ($ds, $drive) = @_;
2754
2755         my $volid = $drive->{file};
2756         return if !$volid;
2757
2758         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2759         if ($storeid) {
2760             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2761             if ($scfg->{disable}) {
2762                 $nodehash = {};
2763             } elsif (my $avail = $scfg->{nodes}) {
2764                 foreach my $node (keys %$nodehash) {
2765                     delete $nodehash->{$node} if !$avail->{$node};
2766                 }
2767             } elsif (!$scfg->{shared}) {
2768                 foreach my $node (keys %$nodehash) {
2769                     delete $nodehash->{$node} if $node ne $nodename
2770                 }
2771             }
2772         }
2773     });
2774
2775     return $nodehash
2776 }
2777
2778 sub check_local_storage_availability {
2779     my ($conf, $storecfg) = @_;
2780
2781     my $nodelist = PVE::Cluster::get_nodelist();
2782     my $nodehash = { map { $_ => {} } @$nodelist };
2783
2784     foreach_drive($conf, sub {
2785         my ($ds, $drive) = @_;
2786
2787         my $volid = $drive->{file};
2788         return if !$volid;
2789
2790         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2791         if ($storeid) {
2792             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2793
2794             if ($scfg->{disable}) {
2795                 foreach my $node (keys %$nodehash) {
2796                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2797                 }
2798             } elsif (my $avail = $scfg->{nodes}) {
2799                 foreach my $node (keys %$nodehash) {
2800                     if (!$avail->{$node}) {
2801                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2802                     }
2803                 }
2804             }
2805         }
2806     });
2807
2808     foreach my $node (values %$nodehash) {
2809         if (my $unavail = $node->{unavailable_storages}) {
2810             $node->{unavailable_storages} = [ sort keys %$unavail ];
2811         }
2812     }
2813
2814     return $nodehash
2815 }
2816
2817 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2818 sub check_running {
2819     my ($vmid, $nocheck, $node) = @_;
2820
2821     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2822     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2823 }
2824
2825 sub vzlist {
2826
2827     my $vzlist = config_list();
2828
2829     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2830
2831     while (defined(my $de = $fd->read)) {
2832         next if $de !~ m/^(\d+)\.pid$/;
2833         my $vmid = $1;
2834         next if !defined($vzlist->{$vmid});
2835         if (my $pid = check_running($vmid)) {
2836             $vzlist->{$vmid}->{pid} = $pid;
2837         }
2838     }
2839
2840     return $vzlist;
2841 }
2842
2843 sub disksize {
2844     my ($storecfg, $conf) = @_;
2845
2846     my $bootdisk = $conf->{bootdisk};
2847     return undef if !$bootdisk;
2848     return undef if !is_valid_drivename($bootdisk);
2849
2850     return undef if !$conf->{$bootdisk};
2851
2852     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2853     return undef if !defined($drive);
2854
2855     return undef if drive_is_cdrom($drive);
2856
2857     my $volid = $drive->{file};
2858     return undef if !$volid;
2859
2860     return $drive->{size};
2861 }
2862
2863 our $vmstatus_return_properties = {
2864     vmid => get_standard_option('pve-vmid'),
2865     status => {
2866         description => "Qemu process status.",
2867         type => 'string',
2868         enum => ['stopped', 'running'],
2869     },
2870     maxmem => {
2871         description => "Maximum memory in bytes.",
2872         type => 'integer',
2873         optional => 1,
2874         renderer => 'bytes',
2875     },
2876     maxdisk => {
2877         description => "Root disk size in bytes.",
2878         type => 'integer',
2879         optional => 1,
2880         renderer => 'bytes',
2881     },
2882     name => {
2883         description => "VM name.",
2884         type => 'string',
2885         optional => 1,
2886     },
2887     qmpstatus => {
2888         description => "Qemu QMP agent status.",
2889         type => 'string',
2890         optional => 1,
2891     },
2892     pid => {
2893         description => "PID of running qemu process.",
2894         type => 'integer',
2895         optional => 1,
2896     },
2897     uptime => {
2898         description => "Uptime.",
2899         type => 'integer',
2900         optional => 1,
2901         renderer => 'duration',
2902     },
2903     cpus => {
2904         description => "Maximum usable CPUs.",
2905         type => 'number',
2906         optional => 1,
2907     },
2908     lock => {
2909         description => "The current config lock, if any.",
2910         type => 'string',
2911         optional => 1,
2912     },
2913     tags => {
2914         description  => "The current configured tags, if any",
2915         type => 'string',
2916         optional => 1,
2917     },
2918 };
2919
2920 my $last_proc_pid_stat;
2921
2922 # get VM status information
2923 # This must be fast and should not block ($full == false)
2924 # We only query KVM using QMP if $full == true (this can be slow)
2925 sub vmstatus {
2926     my ($opt_vmid, $full) = @_;
2927
2928     my $res = {};
2929
2930     my $storecfg = PVE::Storage::config();
2931
2932     my $list = vzlist();
2933     my $defaults = load_defaults();
2934
2935     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2936
2937     my $cpucount = $cpuinfo->{cpus} || 1;
2938
2939     foreach my $vmid (keys %$list) {
2940         next if $opt_vmid && ($vmid ne $opt_vmid);
2941
2942         my $conf = PVE::QemuConfig->load_config($vmid);
2943
2944         my $d = { vmid => $vmid };
2945         $d->{pid} = $list->{$vmid}->{pid};
2946
2947         # fixme: better status?
2948         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2949
2950         my $size = disksize($storecfg, $conf);
2951         if (defined($size)) {
2952             $d->{disk} = 0; # no info available
2953             $d->{maxdisk} = $size;
2954         } else {
2955             $d->{disk} = 0;
2956             $d->{maxdisk} = 0;
2957         }
2958
2959         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2960             * ($conf->{cores} || $defaults->{cores});
2961         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2962         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2963
2964         $d->{name} = $conf->{name} || "VM $vmid";
2965         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2966             : $defaults->{memory}*(1024*1024);
2967
2968         if ($conf->{balloon}) {
2969             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2970             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2971                 : $defaults->{shares};
2972         }
2973
2974         $d->{uptime} = 0;
2975         $d->{cpu} = 0;
2976         $d->{mem} = 0;
2977
2978         $d->{netout} = 0;
2979         $d->{netin} = 0;
2980
2981         $d->{diskread} = 0;
2982         $d->{diskwrite} = 0;
2983
2984         $d->{template} = PVE::QemuConfig->is_template($conf);
2985
2986         $d->{serial} = 1 if conf_has_serial($conf);
2987         $d->{lock} = $conf->{lock} if $conf->{lock};
2988         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2989
2990         $res->{$vmid} = $d;
2991     }
2992
2993     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2994     foreach my $dev (keys %$netdev) {
2995         next if $dev !~ m/^tap([1-9]\d*)i/;
2996         my $vmid = $1;
2997         my $d = $res->{$vmid};
2998         next if !$d;
2999
3000         $d->{netout} += $netdev->{$dev}->{receive};
3001         $d->{netin} += $netdev->{$dev}->{transmit};
3002
3003         if ($full) {
3004             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3005             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3006         }
3007
3008     }
3009
3010     my $ctime = gettimeofday;
3011
3012     foreach my $vmid (keys %$list) {
3013
3014         my $d = $res->{$vmid};
3015         my $pid = $d->{pid};
3016         next if !$pid;
3017
3018         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3019         next if !$pstat; # not running
3020
3021         my $used = $pstat->{utime} + $pstat->{stime};
3022
3023         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3024
3025         if ($pstat->{vsize}) {
3026             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3027         }
3028
3029         my $old = $last_proc_pid_stat->{$pid};
3030         if (!$old) {
3031             $last_proc_pid_stat->{$pid} = {
3032                 time => $ctime,
3033                 used => $used,
3034                 cpu => 0,
3035             };
3036             next;
3037         }
3038
3039         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3040
3041         if ($dtime > 1000) {
3042             my $dutime = $used -  $old->{used};
3043
3044             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3045             $last_proc_pid_stat->{$pid} = {
3046                 time => $ctime,
3047                 used => $used,
3048                 cpu => $d->{cpu},
3049             };
3050         } else {
3051             $d->{cpu} = $old->{cpu};
3052         }
3053     }
3054
3055     return $res if !$full;
3056
3057     my $qmpclient = PVE::QMPClient->new();
3058
3059     my $ballooncb = sub {
3060         my ($vmid, $resp) = @_;
3061
3062         my $info = $resp->{'return'};
3063         return if !$info->{max_mem};
3064
3065         my $d = $res->{$vmid};
3066
3067         # use memory assigned to VM
3068         $d->{maxmem} = $info->{max_mem};
3069         $d->{balloon} = $info->{actual};
3070
3071         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3072             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3073             $d->{freemem} = $info->{free_mem};
3074         }
3075
3076         $d->{ballooninfo} = $info;
3077     };
3078
3079     my $blockstatscb = sub {
3080         my ($vmid, $resp) = @_;
3081         my $data = $resp->{'return'} || [];
3082         my $totalrdbytes = 0;
3083         my $totalwrbytes = 0;
3084
3085         for my $blockstat (@$data) {
3086             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3087             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3088
3089             $blockstat->{device} =~ s/drive-//;
3090             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3091         }
3092         $res->{$vmid}->{diskread} = $totalrdbytes;
3093         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3094     };
3095
3096     my $statuscb = sub {
3097         my ($vmid, $resp) = @_;
3098
3099         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3100         # this fails if ballon driver is not loaded, so this must be
3101         # the last commnand (following command are aborted if this fails).
3102         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3103
3104         my $status = 'unknown';
3105         if (!defined($status = $resp->{'return'}->{status})) {
3106             warn "unable to get VM status\n";
3107             return;
3108         }
3109
3110         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3111     };
3112
3113     foreach my $vmid (keys %$list) {
3114         next if $opt_vmid && ($vmid ne $opt_vmid);
3115         next if !$res->{$vmid}->{pid}; # not running
3116         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3117     }
3118
3119     $qmpclient->queue_execute(undef, 2);
3120
3121     foreach my $vmid (keys %$list) {
3122         next if $opt_vmid && ($vmid ne $opt_vmid);
3123         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3124     }
3125
3126     return $res;
3127 }
3128
3129 sub foreach_drive {
3130     my ($conf, $func, @param) = @_;
3131
3132     foreach my $ds (valid_drive_names()) {
3133         next if !defined($conf->{$ds});
3134
3135         my $drive = parse_drive($ds, $conf->{$ds});
3136         next if !$drive;
3137
3138         &$func($ds, $drive, @param);
3139     }
3140 }
3141
3142 sub foreach_volid {
3143     my ($conf, $func, @param) = @_;
3144
3145     my $volhash = {};
3146
3147     my $test_volid = sub {
3148         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3149
3150         return if !$volid;
3151
3152         $volhash->{$volid}->{cdrom} //= 1;
3153         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3154
3155         $volhash->{$volid}->{replicate} //= 0;
3156         $volhash->{$volid}->{replicate} = 1 if $replicate;
3157
3158         $volhash->{$volid}->{shared} //= 0;
3159         $volhash->{$volid}->{shared} = 1 if $shared;
3160
3161         $volhash->{$volid}->{referenced_in_config} //= 0;
3162         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3163
3164         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3165             if defined($snapname);
3166         $volhash->{$volid}->{size} = $size if $size;
3167     };
3168
3169     foreach_drive($conf, sub {
3170         my ($ds, $drive) = @_;
3171         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3172     });
3173
3174     foreach my $snapname (keys %{$conf->{snapshots}}) {
3175         my $snap = $conf->{snapshots}->{$snapname};
3176         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3177         foreach_drive($snap, sub {
3178             my ($ds, $drive) = @_;
3179             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3180         });
3181     }
3182
3183     foreach my $volid (keys %$volhash) {
3184         &$func($volid, $volhash->{$volid}, @param);
3185     }
3186 }
3187
3188 sub conf_has_serial {
3189     my ($conf) = @_;
3190
3191     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3192         if ($conf->{"serial$i"}) {
3193             return 1;
3194         }
3195     }
3196
3197     return 0;
3198 }
3199
3200 sub conf_has_audio {
3201     my ($conf, $id) = @_;
3202
3203     $id //= 0;
3204     my $audio = $conf->{"audio$id"};
3205     return undef if !defined($audio);
3206
3207     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3208     my $audiodriver = $audioproperties->{driver} // 'spice';
3209
3210     return {
3211         dev => $audioproperties->{device},
3212         dev_id => "audiodev$id",
3213         backend => $audiodriver,
3214         backend_id => "$audiodriver-backend${id}",
3215     };
3216 }
3217
3218 sub vga_conf_has_spice {
3219     my ($vga) = @_;
3220
3221     my $vgaconf = parse_vga($vga);
3222     my $vgatype = $vgaconf->{type};
3223     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3224
3225     return $1 || 1;
3226 }
3227
3228 sub is_native($) {
3229     my ($arch) = @_;
3230     return get_host_arch() eq $arch;
3231 }
3232
3233 sub get_vm_arch {
3234     my ($conf) = @_;
3235     return $conf->{arch} // get_host_arch();
3236 }
3237
3238 my $default_machines = {
3239     x86_64 => 'pc',
3240     aarch64 => 'virt',
3241 };
3242
3243 sub get_vm_machine {
3244     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3245
3246     my $machine = $forcemachine || $conf->{machine};
3247
3248     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3249         $arch //= 'x86_64';
3250         $machine ||= $default_machines->{$arch};
3251         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3252     }
3253
3254     return $machine;
3255 }
3256
3257 sub get_ovmf_files($) {
3258     my ($arch) = @_;
3259
3260     my $ovmf = $OVMF->{$arch}
3261         or die "no OVMF images known for architecture '$arch'\n";
3262
3263     return @$ovmf;
3264 }
3265
3266 my $Arch2Qemu = {
3267     aarch64 => '/usr/bin/qemu-system-aarch64',
3268     x86_64 => '/usr/bin/qemu-system-x86_64',
3269 };
3270 sub get_command_for_arch($) {
3271     my ($arch) = @_;
3272     return '/usr/bin/kvm' if is_native($arch);
3273
3274     my $cmd = $Arch2Qemu->{$arch}
3275         or die "don't know how to emulate architecture '$arch'\n";
3276     return $cmd;
3277 }
3278
3279 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3280 # to use in a QEMU command line (-cpu element), first array_intersect the result
3281 # of query_supported_ with query_understood_. This is necessary because:
3282 #
3283 # a) query_understood_ returns flags the host cannot use and
3284 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3285 #    flags, but CPU settings - with most of them being flags. Those settings
3286 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3287 #
3288 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3289 # expensive. If you need the value returned from this, you can get it much
3290 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3291 # $accel being 'kvm' or 'tcg'.
3292 #
3293 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3294 # changes, automatically populating pmxcfs.
3295 #
3296 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3297 # since kvm and tcg machines support different flags
3298 #
3299 sub query_supported_cpu_flags {
3300     my ($arch) = @_;
3301
3302     $arch //= get_host_arch();
3303     my $default_machine = $default_machines->{$arch};
3304
3305     my $flags = {};
3306
3307     # FIXME: Once this is merged, the code below should work for ARM as well:
3308     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3309     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3310         $arch eq "aarch64";
3311
3312     my $kvm_supported = defined(kvm_version());
3313     my $qemu_cmd = get_command_for_arch($arch);
3314     my $fakevmid = -1;
3315     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3316
3317     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3318     my $query_supported_run_qemu = sub {
3319         my ($kvm) = @_;
3320
3321         my $flags = {};
3322         my $cmd = [
3323             $qemu_cmd,
3324             '-machine', $default_machine,
3325             '-display', 'none',
3326             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3327             '-mon', 'chardev=qmp,mode=control',
3328             '-pidfile', $pidfile,
3329             '-S', '-daemonize'
3330         ];
3331
3332         if (!$kvm) {
3333             push @$cmd, '-accel', 'tcg';
3334         }
3335
3336         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3337         die "QEMU flag querying VM exited with code " . $rc if $rc;
3338
3339         eval {
3340             my $cmd_result = mon_cmd(
3341                 $fakevmid,
3342                 'query-cpu-model-expansion',
3343                 type => 'full',
3344                 model => { name => 'host' }
3345             );
3346
3347             my $props = $cmd_result->{model}->{props};
3348             foreach my $prop (keys %$props) {
3349                 next if $props->{$prop} ne '1';
3350                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3351                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3352                 # We only keep those with underscores, to match /proc/cpuinfo
3353                 $prop =~ s/\.|-/_/g;
3354                 $flags->{$prop} = 1;
3355             }
3356         };
3357         my $err = $@;
3358
3359         # force stop with 10 sec timeout and 'nocheck'
3360         # always stop, even if QMP failed
3361         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3362
3363         die $err if $err;
3364
3365         return [ sort keys %$flags ];
3366     };
3367
3368     # We need to query QEMU twice, since KVM and TCG have different supported flags
3369     PVE::QemuConfig->lock_config($fakevmid, sub {
3370         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3371         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3372
3373         if ($kvm_supported) {
3374             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3375             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3376         }
3377     });
3378
3379     return $flags;
3380 }
3381
3382 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3383 my $understood_cpu_flag_dir = "/usr/share/kvm";
3384 sub query_understood_cpu_flags {
3385     my $arch = get_host_arch();
3386     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3387
3388     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3389         if ! -e $filepath;
3390
3391     my $raw = file_get_contents($filepath);
3392     $raw =~ s/^\s+|\s+$//g;
3393     my @flags = split(/\s+/, $raw);
3394
3395     return \@flags;
3396 }
3397
3398 sub config_to_command {
3399     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3400
3401     my $cmd = [];
3402     my $globalFlags = [];
3403     my $machineFlags = [];
3404     my $rtcFlags = [];
3405     my $devices = [];
3406     my $pciaddr = '';
3407     my $bridges = {};
3408     my $vernum = 0; # unknown
3409     my $ostype = $conf->{ostype};
3410     my $winversion = windows_version($ostype);
3411     my $kvm = $conf->{kvm};
3412     my $nodename = nodename();
3413
3414     my $arch = get_vm_arch($conf);
3415     my $kvm_binary = get_command_for_arch($arch);
3416     my $kvmver = kvm_user_version($kvm_binary);
3417
3418     my $add_pve_version = min_version($kvmver, 4, 1);
3419
3420     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3421     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3422     $kvm //= 1 if is_native($arch);
3423
3424     $machine_version =~ m/(\d+)\.(\d+)/;
3425     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3426         if !PVE::QemuServer::min_version($kvmver, $1, $2);
3427
3428     if ($kvm) {
3429         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3430             if !defined kvm_version();
3431     }
3432
3433     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3434         $vernum = $1*1000000+$2*1000;
3435     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3436         $vernum = $1*1000000+$2*1000+$3;
3437     }
3438
3439     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3440
3441     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3442     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3443     my $use_old_bios_files = undef;
3444     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3445
3446     my $cpuunits = defined($conf->{cpuunits}) ?
3447             $conf->{cpuunits} : $defaults->{cpuunits};
3448
3449     push @$cmd, $kvm_binary;
3450
3451     push @$cmd, '-id', $vmid;
3452
3453     my $vmname = $conf->{name} || "vm$vmid";
3454
3455     push @$cmd, '-name', $vmname;
3456
3457     my $use_virtio = 0;
3458
3459     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3460     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3461     push @$cmd, '-mon', "chardev=qmp,mode=control";
3462
3463     if (min_version($machine_version, 2, 12)) {
3464         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3465         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3466     }
3467
3468     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3469
3470     push @$cmd, '-daemonize';
3471
3472     if ($conf->{smbios1}) {
3473         my $smbios_conf = parse_smbios1($conf->{smbios1});
3474         if ($smbios_conf->{base64}) {
3475             # Do not pass base64 flag to qemu
3476             delete $smbios_conf->{base64};
3477             my $smbios_string = "";
3478             foreach my $key (keys %$smbios_conf) {
3479                 my $value;
3480                 if ($key eq "uuid") {
3481                     $value = $smbios_conf->{uuid}
3482                 } else {
3483                     $value = decode_base64($smbios_conf->{$key});
3484                 }
3485                 # qemu accepts any binary data, only commas need escaping by double comma
3486                 $value =~ s/,/,,/g;
3487                 $smbios_string .= "," . $key . "=" . $value if $value;
3488             }
3489             push @$cmd, '-smbios', "type=1" . $smbios_string;
3490         } else {
3491             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3492         }
3493     }
3494
3495     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3496     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3497         die "uefi base image not found\n" if ! -f $ovmf_code;
3498
3499         my $path;
3500         my $format;
3501         if (my $efidisk = $conf->{efidisk0}) {
3502             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3503             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3504             $format = $d->{format};
3505             if ($storeid) {
3506                 $path = PVE::Storage::path($storecfg, $d->{file});
3507                 if (!defined($format)) {
3508                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3509                     $format = qemu_img_format($scfg, $volname);
3510                 }
3511             } else {
3512                 $path = $d->{file};
3513                 die "efidisk format must be specified\n"
3514                     if !defined($format);
3515             }
3516         } else {
3517             warn "no efidisk configured! Using temporary efivars disk.\n";
3518             $path = "/tmp/$vmid-ovmf.fd";
3519             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3520             $format = 'raw';
3521         }
3522
3523         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3524         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3525     }
3526
3527     # load q35 config
3528     if ($q35) {
3529         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3530         if (min_version($machine_version, 4, 0)) {
3531             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3532         } else {
3533             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3534         }
3535     }
3536
3537     if ($conf->{vmgenid}) {
3538         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3539     }
3540
3541     # add usb controllers
3542     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3543     push @$devices, @usbcontrollers if @usbcontrollers;
3544     my $vga = parse_vga($conf->{vga});
3545
3546     my $qxlnum = vga_conf_has_spice($conf->{vga});
3547     $vga->{type} = 'qxl' if $qxlnum;
3548
3549     if (!$vga->{type}) {
3550         if ($arch eq 'aarch64') {
3551             $vga->{type} = 'virtio';
3552         } elsif (min_version($machine_version, 2, 9)) {
3553             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3554         } else {
3555             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3556         }
3557     }
3558
3559     # enable absolute mouse coordinates (needed by vnc)
3560     my $tablet;
3561     if (defined($conf->{tablet})) {
3562         $tablet = $conf->{tablet};
3563     } else {
3564         $tablet = $defaults->{tablet};
3565         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3566         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3567     }
3568
3569     if ($tablet) {
3570         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3571         my $kbd = print_keyboarddevice_full($conf, $arch);
3572         push @$devices, '-device', $kbd if defined($kbd);
3573     }
3574
3575     my $kvm_off = 0;
3576     my $gpu_passthrough;
3577
3578     # host pci devices
3579     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3580         my $id = "hostpci$i";
3581         my $d = parse_hostpci($conf->{$id});
3582         next if !$d;
3583
3584         if (my $pcie = $d->{pcie}) {
3585             die "q35 machine model is not enabled" if !$q35;
3586             # win7 wants to have the pcie devices directly on the pcie bus
3587             # instead of in the root port
3588             if ($winversion == 7) {
3589                 $pciaddr = print_pcie_addr("${id}bus0");
3590             } else {
3591                 # add more root ports if needed, 4 are present by default
3592                 # by pve-q35 cfgs, rest added here on demand.
3593                 if ($i > 3) {
3594                     push @$devices, '-device', print_pcie_root_port($i);
3595                 }
3596                 $pciaddr = print_pcie_addr($id);
3597             }
3598         } else {
3599             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3600         }
3601
3602         my $xvga = '';
3603         if ($d->{'x-vga'}) {
3604             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3605             $kvm_off = 1;
3606             $vga->{type} = 'none' if !defined($conf->{vga});
3607             $gpu_passthrough = 1;
3608         }
3609
3610         my $pcidevices = $d->{pciid};
3611         my $multifunction = 1 if @$pcidevices > 1;
3612
3613         my $sysfspath;
3614         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3615             my $pci_id = $pcidevices->[0]->{id};
3616             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3617             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3618         } elsif ($d->{mdev}) {
3619             warn "ignoring mediated device '$id' with multifunction device\n";
3620         }
3621
3622         my $j=0;
3623         foreach my $pcidevice (@$pcidevices) {
3624             my $devicestr = "vfio-pci";
3625
3626             if ($sysfspath) {
3627                 $devicestr .= ",sysfsdev=$sysfspath";
3628             } else {
3629                 $devicestr .= ",host=$pcidevice->{id}";
3630             }
3631
3632             my $mf_addr = $multifunction ? ".$j" : '';
3633             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3634
3635             if ($j == 0) {
3636                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3637                 $devicestr .= "$xvga";
3638                 $devicestr .= ",multifunction=on" if $multifunction;
3639                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3640             }
3641
3642             push @$devices, '-device', $devicestr;
3643             $j++;
3644         }
3645     }
3646
3647     # usb devices
3648     my $usb_dev_features = {};
3649     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3650
3651     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3652     push @$devices, @usbdevices if @usbdevices;
3653     # serial devices
3654     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3655         if (my $path = $conf->{"serial$i"}) {
3656             if ($path eq 'socket') {
3657                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3658                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3659                 # On aarch64, serial0 is the UART device. Qemu only allows
3660                 # connecting UART devices via the '-serial' command line, as
3661                 # the device has a fixed slot on the hardware...
3662                 if ($arch eq 'aarch64' && $i == 0) {
3663                     push @$devices, '-serial', "chardev:serial$i";
3664                 } else {
3665                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3666                 }
3667             } else {
3668                 die "no such serial device\n" if ! -c $path;
3669                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3670                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3671             }
3672         }
3673     }
3674
3675     # parallel devices
3676     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3677         if (my $path = $conf->{"parallel$i"}) {
3678             die "no such parallel device\n" if ! -c $path;
3679             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3680             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3681             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3682         }
3683     }
3684
3685     if (my $audio = conf_has_audio($conf)) {
3686
3687         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3688
3689         my $id = $audio->{dev_id};
3690         if ($audio->{dev} eq 'AC97') {
3691             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3692         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3693             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3694             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3695             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3696         } else {
3697             die "unkown audio device '$audio->{dev}', implement me!";
3698         }
3699
3700         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3701     }
3702
3703     my $sockets = 1;
3704     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3705     $sockets = $conf->{sockets} if  $conf->{sockets};
3706
3707     my $cores = $conf->{cores} || 1;
3708
3709     my $maxcpus = $sockets * $cores;
3710
3711     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3712
3713     my $allowed_vcpus = $cpuinfo->{cpus};
3714
3715     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3716         if ($allowed_vcpus < $maxcpus);
3717
3718     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3719
3720         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3721         for (my $i = 2; $i <= $vcpus; $i++)  {
3722             my $cpustr = print_cpu_device($conf,$i);
3723             push @$cmd, '-device', $cpustr;
3724         }
3725
3726     } else {
3727
3728         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3729     }
3730     push @$cmd, '-nodefaults';
3731
3732     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3733
3734     my $bootindex_hash = {};
3735     my $i = 1;
3736     foreach my $o (split(//, $bootorder)) {
3737         $bootindex_hash->{$o} = $i*100;
3738         $i++;
3739     }
3740
3741     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3742
3743     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3744
3745     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3746
3747     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3748         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3749         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3750         push @$cmd,  '-vnc', "unix:$socket,password";
3751     } else {
3752         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3753         push @$cmd, '-nographic';
3754     }
3755
3756     # time drift fix
3757     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3758
3759     my $useLocaltime = $conf->{localtime};
3760
3761     if ($winversion >= 5) { # windows
3762         $useLocaltime = 1 if !defined($conf->{localtime});
3763
3764         # use time drift fix when acpi is enabled
3765         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3766             $tdf = 1 if !defined($conf->{tdf});
3767         }
3768     }
3769
3770     if ($winversion >= 6) {
3771         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3772         push @$cmd, '-no-hpet';
3773     }
3774
3775     push @$rtcFlags, 'driftfix=slew' if $tdf;
3776
3777     if (!$kvm) {
3778         push @$machineFlags, 'accel=tcg';
3779     }
3780
3781     if ($machine_type) {
3782         push @$machineFlags, "type=${machine_type}";
3783     }
3784
3785     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3786         push @$rtcFlags, "base=$conf->{startdate}";
3787     } elsif ($useLocaltime) {
3788         push @$rtcFlags, 'base=localtime';
3789     }
3790
3791     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3792
3793     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3794
3795     push @$cmd, '-S' if $conf->{freeze};
3796
3797     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3798
3799     my $guest_agent = parse_guest_agent($conf);
3800
3801     if ($guest_agent->{enabled}) {
3802         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3803         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3804
3805         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3806             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3807             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3808             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3809         } elsif ($guest_agent->{type} eq 'isa') {
3810             push @$devices, '-device', "isa-serial,chardev=qga0";
3811         }
3812     }
3813
3814     my $spice_port;
3815
3816     if ($qxlnum) {
3817         if ($qxlnum > 1) {
3818             if ($winversion){
3819                 for(my $i = 1; $i < $qxlnum; $i++){
3820                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3821                 }
3822             } else {
3823                 # assume other OS works like Linux
3824                 my ($ram, $vram) = ("134217728", "67108864");
3825                 if ($vga->{memory}) {
3826                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3827                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3828                 }
3829                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3830                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3831             }
3832         }
3833
3834         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3835
3836         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3837         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3838         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3839
3840         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3841         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3842         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3843
3844         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3845         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3846
3847         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3848         if ($spice_enhancement->{foldersharing}) {
3849             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3850             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3851         }
3852
3853         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3854         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3855         push @$devices, '-spice', "$spice_opts";
3856     }
3857
3858     # enable balloon by default, unless explicitly disabled
3859     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3860         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3861         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3862     }
3863
3864     if ($conf->{watchdog}) {
3865         my $wdopts = parse_watchdog($conf->{watchdog});
3866         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3867         my $watchdog = $wdopts->{model} || 'i6300esb';
3868         push @$devices, '-device', "$watchdog$pciaddr";
3869         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3870     }
3871
3872     my $vollist = [];
3873     my $scsicontroller = {};
3874     my $ahcicontroller = {};
3875     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3876
3877     # Add iscsi initiator name if available
3878     if (my $initiator = get_initiator_name()) {
3879         push @$devices, '-iscsi', "initiator-name=$initiator";
3880     }
3881
3882     foreach_drive($conf, sub {
3883         my ($ds, $drive) = @_;
3884
3885         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3886             push @$vollist, $drive->{file};
3887         }
3888
3889         # ignore efidisk here, already added in bios/fw handling code above
3890         return if $drive->{interface} eq 'efidisk';
3891
3892         $use_virtio = 1 if $ds =~ m/^virtio/;
3893
3894         if (drive_is_cdrom ($drive)) {
3895             if ($bootindex_hash->{d}) {
3896                 $drive->{bootindex} = $bootindex_hash->{d};
3897                 $bootindex_hash->{d} += 1;
3898             }
3899         } else {
3900             if ($bootindex_hash->{c}) {
3901                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3902                 $bootindex_hash->{c} += 1;
3903             }
3904         }
3905
3906         if($drive->{interface} eq 'virtio'){
3907            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3908         }
3909
3910         if ($drive->{interface} eq 'scsi') {
3911
3912             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3913
3914             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3915             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3916
3917             my $iothread = '';
3918             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3919                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3920                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3921             } elsif ($drive->{iothread}) {
3922                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3923             }
3924
3925             my $queues = '';
3926             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3927                 $queues = ",num_queues=$drive->{queues}";
3928             }
3929
3930             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3931             $scsicontroller->{$controller}=1;
3932         }
3933
3934         if ($drive->{interface} eq 'sata') {
3935            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3936            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3937            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3938            $ahcicontroller->{$controller}=1;
3939         }
3940
3941         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3942         push @$devices, '-drive',$drive_cmd;
3943         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3944     });
3945
3946     for (my $i = 0; $i < $MAX_NETS; $i++) {
3947          next if !$conf->{"net$i"};
3948          my $d = parse_net($conf->{"net$i"});
3949          next if !$d;
3950
3951          $use_virtio = 1 if $d->{model} eq 'virtio';
3952
3953          if ($bootindex_hash->{n}) {
3954             $d->{bootindex} = $bootindex_hash->{n};
3955             $bootindex_hash->{n} += 1;
3956          }
3957
3958          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3959          push @$devices, '-netdev', $netdevfull;
3960
3961          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3962          push @$devices, '-device', $netdevicefull;
3963     }
3964
3965     if ($conf->{ivshmem}) {
3966         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3967
3968         my $bus;
3969         if ($q35) {
3970             $bus = print_pcie_addr("ivshmem");
3971         } else {
3972             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3973         }
3974
3975         my $ivshmem_name = $ivshmem->{name} // $vmid;
3976         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3977
3978         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3979         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3980     }
3981
3982     # pci.4 is nested in pci.1
3983     $bridges->{1} = 1 if $bridges->{4};
3984
3985     if (!$q35) {
3986         # add pci bridges
3987         if (min_version($machine_version, 2, 3)) {
3988            $bridges->{1} = 1;
3989            $bridges->{2} = 1;
3990         }
3991
3992         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3993
3994     }
3995
3996     for my $k (sort {$b cmp $a} keys %$bridges) {
3997         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3998         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3999         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4000         if ($q35) {
4001             # add after -readconfig pve-q35.cfg
4002             splice @$devices, 2, 0, '-device', $devstr;
4003         } else {
4004             unshift @$devices, '-device', $devstr if $k > 0;
4005         }
4006     }
4007
4008     push @$cmd, @$devices;
4009     push @$cmd, '-rtc', join(',', @$rtcFlags)
4010         if scalar(@$rtcFlags);
4011     push @$cmd, '-machine', join(',', @$machineFlags)
4012         if scalar(@$machineFlags);
4013     push @$cmd, '-global', join(',', @$globalFlags)
4014         if scalar(@$globalFlags);
4015
4016     if (my $vmstate = $conf->{vmstate}) {
4017         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4018         push @$vollist, $vmstate;
4019         push @$cmd, '-loadstate', $statepath;
4020         print "activating and using '$vmstate' as vmstate\n";
4021     }
4022
4023     # add custom args
4024     if ($conf->{args}) {
4025         my $aa = PVE::Tools::split_args($conf->{args});
4026         push @$cmd, @$aa;
4027     }
4028
4029     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4030 }
4031
4032 sub spice_port {
4033     my ($vmid) = @_;
4034
4035     my $res = mon_cmd($vmid, 'query-spice');
4036
4037     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4038 }
4039
4040 sub vm_devices_list {
4041     my ($vmid) = @_;
4042
4043     my $res = mon_cmd($vmid, 'query-pci');
4044     my $devices_to_check = [];
4045     my $devices = {};
4046     foreach my $pcibus (@$res) {
4047         push @$devices_to_check, @{$pcibus->{devices}},
4048     }
4049
4050     while (@$devices_to_check) {
4051         my $to_check = [];
4052         for my $d (@$devices_to_check) {
4053             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4054             next if !$d->{'pci_bridge'};
4055
4056             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4057             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4058         }
4059         $devices_to_check = $to_check;
4060     }
4061
4062     my $resblock = mon_cmd($vmid, 'query-block');
4063     foreach my $block (@$resblock) {
4064         if($block->{device} =~ m/^drive-(\S+)/){
4065                 $devices->{$1} = 1;
4066         }
4067     }
4068
4069     my $resmice = mon_cmd($vmid, 'query-mice');
4070     foreach my $mice (@$resmice) {
4071         if ($mice->{name} eq 'QEMU HID Tablet') {
4072             $devices->{tablet} = 1;
4073             last;
4074         }
4075     }
4076
4077     # for usb devices there is no query-usb
4078     # but we can iterate over the entries in
4079     # qom-list path=/machine/peripheral
4080     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4081     foreach my $per (@$resperipheral) {
4082         if ($per->{name} =~ m/^usb\d+$/) {
4083             $devices->{$per->{name}} = 1;
4084         }
4085     }
4086
4087     return $devices;
4088 }
4089
4090 sub vm_deviceplug {
4091     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4092
4093     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4094
4095     my $devices_list = vm_devices_list($vmid);
4096     return 1 if defined($devices_list->{$deviceid});
4097
4098     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4099
4100     if ($deviceid eq 'tablet') {
4101
4102         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4103
4104     } elsif ($deviceid eq 'keyboard') {
4105
4106         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4107
4108     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4109
4110         die "usb hotplug currently not reliable\n";
4111         # since we can't reliably hot unplug all added usb devices
4112         # and usb passthrough disables live migration
4113         # we disable usb hotplugging for now
4114         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4115
4116     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4117
4118         qemu_iothread_add($vmid, $deviceid, $device);
4119
4120         qemu_driveadd($storecfg, $vmid, $device);
4121         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4122
4123         qemu_deviceadd($vmid, $devicefull);
4124         eval { qemu_deviceaddverify($vmid, $deviceid); };
4125         if (my $err = $@) {
4126             eval { qemu_drivedel($vmid, $deviceid); };
4127             warn $@ if $@;
4128             die $err;
4129         }
4130
4131     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4132
4133
4134         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4135         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4136         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4137
4138         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4139
4140         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4141             qemu_iothread_add($vmid, $deviceid, $device);
4142             $devicefull .= ",iothread=iothread-$deviceid";
4143         }
4144
4145         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4146             $devicefull .= ",num_queues=$device->{queues}";
4147         }
4148
4149         qemu_deviceadd($vmid, $devicefull);
4150         qemu_deviceaddverify($vmid, $deviceid);
4151
4152     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4153
4154         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4155         qemu_driveadd($storecfg, $vmid, $device);
4156
4157         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4158         eval { qemu_deviceadd($vmid, $devicefull); };
4159         if (my $err = $@) {
4160             eval { qemu_drivedel($vmid, $deviceid); };
4161             warn $@ if $@;
4162             die $err;
4163         }
4164
4165     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4166
4167         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4168
4169         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4170         my $use_old_bios_files = undef;
4171         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4172
4173         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4174         qemu_deviceadd($vmid, $netdevicefull);
4175         eval {
4176             qemu_deviceaddverify($vmid, $deviceid);
4177             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4178         };
4179         if (my $err = $@) {
4180             eval { qemu_netdevdel($vmid, $deviceid); };
4181             warn $@ if $@;
4182             die $err;
4183         }
4184
4185     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4186
4187         my $bridgeid = $2;
4188         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4189         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4190
4191         qemu_deviceadd($vmid, $devicefull);
4192         qemu_deviceaddverify($vmid, $deviceid);
4193
4194     } else {
4195         die "can't hotplug device '$deviceid'\n";
4196     }
4197
4198     return 1;
4199 }
4200
4201 # fixme: this should raise exceptions on error!
4202 sub vm_deviceunplug {
4203     my ($vmid, $conf, $deviceid) = @_;
4204
4205     my $devices_list = vm_devices_list($vmid);
4206     return 1 if !defined($devices_list->{$deviceid});
4207
4208     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4209
4210     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4211
4212         qemu_devicedel($vmid, $deviceid);
4213
4214     } elsif ($deviceid =~ m/^usb\d+$/) {
4215
4216         die "usb hotplug currently not reliable\n";
4217         # when unplugging usb devices this way,
4218         # there may be remaining usb controllers/hubs
4219         # so we disable it for now
4220         qemu_devicedel($vmid, $deviceid);
4221         qemu_devicedelverify($vmid, $deviceid);
4222
4223     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4224
4225         qemu_devicedel($vmid, $deviceid);
4226         qemu_devicedelverify($vmid, $deviceid);
4227         qemu_drivedel($vmid, $deviceid);
4228         qemu_iothread_del($conf, $vmid, $deviceid);
4229
4230     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4231
4232         qemu_devicedel($vmid, $deviceid);
4233         qemu_devicedelverify($vmid, $deviceid);
4234         qemu_iothread_del($conf, $vmid, $deviceid);
4235
4236     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4237
4238         qemu_devicedel($vmid, $deviceid);
4239         qemu_drivedel($vmid, $deviceid);
4240         qemu_deletescsihw($conf, $vmid, $deviceid);
4241
4242     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4243
4244         qemu_devicedel($vmid, $deviceid);
4245         qemu_devicedelverify($vmid, $deviceid);
4246         qemu_netdevdel($vmid, $deviceid);
4247
4248     } else {
4249         die "can't unplug device '$deviceid'\n";
4250     }
4251
4252     return 1;
4253 }
4254
4255 sub qemu_deviceadd {
4256     my ($vmid, $devicefull) = @_;
4257
4258     $devicefull = "driver=".$devicefull;
4259     my %options =  split(/[=,]/, $devicefull);
4260
4261     mon_cmd($vmid, "device_add" , %options);
4262 }
4263
4264 sub qemu_devicedel {
4265     my ($vmid, $deviceid) = @_;
4266
4267     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4268 }
4269
4270 sub qemu_iothread_add {
4271     my($vmid, $deviceid, $device) = @_;
4272
4273     if ($device->{iothread}) {
4274         my $iothreads = vm_iothreads_list($vmid);
4275         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4276     }
4277 }
4278
4279 sub qemu_iothread_del {
4280     my($conf, $vmid, $deviceid) = @_;
4281
4282     my $confid = $deviceid;
4283     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4284         $confid = 'scsi' . $1;
4285     }
4286     my $device = parse_drive($confid, $conf->{$confid});
4287     if ($device->{iothread}) {
4288         my $iothreads = vm_iothreads_list($vmid);
4289         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4290     }
4291 }
4292
4293 sub qemu_objectadd {
4294     my($vmid, $objectid, $qomtype) = @_;
4295
4296     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4297
4298     return 1;
4299 }
4300
4301 sub qemu_objectdel {
4302     my($vmid, $objectid) = @_;
4303
4304     mon_cmd($vmid, "object-del", id => $objectid);
4305
4306     return 1;
4307 }
4308
4309 sub qemu_driveadd {
4310     my ($storecfg, $vmid, $device) = @_;
4311
4312     my $drive = print_drive_full($storecfg, $vmid, $device);
4313     $drive =~ s/\\/\\\\/g;
4314     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4315
4316     # If the command succeeds qemu prints: "OK"
4317     return 1 if $ret =~ m/OK/s;
4318
4319     die "adding drive failed: $ret\n";
4320 }
4321
4322 sub qemu_drivedel {
4323     my($vmid, $deviceid) = @_;
4324
4325     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4326     $ret =~ s/^\s+//;
4327
4328     return 1 if $ret eq "";
4329
4330     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4331     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4332
4333     die "deleting drive $deviceid failed : $ret\n";
4334 }
4335
4336 sub qemu_deviceaddverify {
4337     my ($vmid, $deviceid) = @_;
4338
4339     for (my $i = 0; $i <= 5; $i++) {
4340          my $devices_list = vm_devices_list($vmid);
4341          return 1 if defined($devices_list->{$deviceid});
4342          sleep 1;
4343     }
4344
4345     die "error on hotplug device '$deviceid'\n";
4346 }
4347
4348
4349 sub qemu_devicedelverify {
4350     my ($vmid, $deviceid) = @_;
4351
4352     # need to verify that the device is correctly removed as device_del
4353     # is async and empty return is not reliable
4354
4355     for (my $i = 0; $i <= 5; $i++) {
4356          my $devices_list = vm_devices_list($vmid);
4357          return 1 if !defined($devices_list->{$deviceid});
4358          sleep 1;
4359     }
4360
4361     die "error on hot-unplugging device '$deviceid'\n";
4362 }
4363
4364 sub qemu_findorcreatescsihw {
4365     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4366
4367     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4368
4369     my $scsihwid="$controller_prefix$controller";
4370     my $devices_list = vm_devices_list($vmid);
4371
4372     if(!defined($devices_list->{$scsihwid})) {
4373         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4374     }
4375
4376     return 1;
4377 }
4378
4379 sub qemu_deletescsihw {
4380     my ($conf, $vmid, $opt) = @_;
4381
4382     my $device = parse_drive($opt, $conf->{$opt});
4383
4384     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4385         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4386         return 1;
4387     }
4388
4389     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4390
4391     my $devices_list = vm_devices_list($vmid);
4392     foreach my $opt (keys %{$devices_list}) {
4393         if (PVE::QemuServer::is_valid_drivename($opt)) {
4394             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4395             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4396                 return 1;
4397             }
4398         }
4399     }
4400
4401     my $scsihwid="scsihw$controller";
4402
4403     vm_deviceunplug($vmid, $conf, $scsihwid);
4404
4405     return 1;
4406 }
4407
4408 sub qemu_add_pci_bridge {
4409     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4410
4411     my $bridges = {};
4412
4413     my $bridgeid;
4414
4415     print_pci_addr($device, $bridges, $arch, $machine_type);
4416
4417     while (my ($k, $v) = each %$bridges) {
4418         $bridgeid = $k;
4419     }
4420     return 1 if !defined($bridgeid) || $bridgeid < 1;
4421
4422     my $bridge = "pci.$bridgeid";
4423     my $devices_list = vm_devices_list($vmid);
4424
4425     if (!defined($devices_list->{$bridge})) {
4426         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4427     }
4428
4429     return 1;
4430 }
4431
4432 sub qemu_set_link_status {
4433     my ($vmid, $device, $up) = @_;
4434
4435     mon_cmd($vmid, "set_link", name => $device,
4436                up => $up ? JSON::true : JSON::false);
4437 }
4438
4439 sub qemu_netdevadd {
4440     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4441
4442     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4443     my %options =  split(/[=,]/, $netdev);
4444
4445     mon_cmd($vmid, "netdev_add",  %options);
4446     return 1;
4447 }
4448
4449 sub qemu_netdevdel {
4450     my ($vmid, $deviceid) = @_;
4451
4452     mon_cmd($vmid, "netdev_del", id => $deviceid);
4453 }
4454
4455 sub qemu_usb_hotplug {
4456     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4457
4458     return if !$device;
4459
4460     # remove the old one first
4461     vm_deviceunplug($vmid, $conf, $deviceid);
4462
4463     # check if xhci controller is necessary and available
4464     if ($device->{usb3}) {
4465
4466         my $devicelist = vm_devices_list($vmid);
4467
4468         if (!$devicelist->{xhci}) {
4469             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4470             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4471         }
4472     }
4473     my $d = parse_usb_device($device->{host});
4474     $d->{usb3} = $device->{usb3};
4475
4476     # add the new one
4477     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4478 }
4479
4480 sub qemu_cpu_hotplug {
4481     my ($vmid, $conf, $vcpus) = @_;
4482
4483     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4484
4485     my $sockets = 1;
4486     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4487     $sockets = $conf->{sockets} if  $conf->{sockets};
4488     my $cores = $conf->{cores} || 1;
4489     my $maxcpus = $sockets * $cores;
4490
4491     $vcpus = $maxcpus if !$vcpus;
4492
4493     die "you can't add more vcpus than maxcpus\n"
4494         if $vcpus > $maxcpus;
4495
4496     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4497
4498     if ($vcpus < $currentvcpus) {
4499
4500         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4501
4502             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4503                 qemu_devicedel($vmid, "cpu$i");
4504                 my $retry = 0;
4505                 my $currentrunningvcpus = undef;
4506                 while (1) {
4507                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4508                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4509                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4510                     $retry++;
4511                     sleep 1;
4512                 }
4513                 #update conf after each succesfull cpu unplug
4514                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4515                 PVE::QemuConfig->write_config($vmid, $conf);
4516             }
4517         } else {
4518             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4519         }
4520
4521         return;
4522     }
4523
4524     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4525     die "vcpus in running vm does not match its configuration\n"
4526         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4527
4528     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4529
4530         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4531             my $cpustr = print_cpu_device($conf, $i);
4532             qemu_deviceadd($vmid, $cpustr);
4533
4534             my $retry = 0;
4535             my $currentrunningvcpus = undef;
4536             while (1) {
4537                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4538                 last if scalar(@{$currentrunningvcpus}) == $i;
4539                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4540                 sleep 1;
4541                 $retry++;
4542             }
4543             #update conf after each succesfull cpu hotplug
4544             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4545             PVE::QemuConfig->write_config($vmid, $conf);
4546         }
4547     } else {
4548
4549         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4550             mon_cmd($vmid, "cpu-add", id => int($i));
4551         }
4552     }
4553 }
4554
4555 sub qemu_block_set_io_throttle {
4556     my ($vmid, $deviceid,
4557         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4558         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4559         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4560         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4561
4562     return if !check_running($vmid) ;
4563
4564     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4565         bps => int($bps),
4566         bps_rd => int($bps_rd),
4567         bps_wr => int($bps_wr),
4568         iops => int($iops),
4569         iops_rd => int($iops_rd),
4570         iops_wr => int($iops_wr),
4571         bps_max => int($bps_max),
4572         bps_rd_max => int($bps_rd_max),
4573         bps_wr_max => int($bps_wr_max),
4574         iops_max => int($iops_max),
4575         iops_rd_max => int($iops_rd_max),
4576         iops_wr_max => int($iops_wr_max),
4577         bps_max_length => int($bps_max_length),
4578         bps_rd_max_length => int($bps_rd_max_length),
4579         bps_wr_max_length => int($bps_wr_max_length),
4580         iops_max_length => int($iops_max_length),
4581         iops_rd_max_length => int($iops_rd_max_length),
4582         iops_wr_max_length => int($iops_wr_max_length),
4583     );
4584
4585 }
4586
4587 # old code, only used to shutdown old VM after update
4588 sub __read_avail {
4589     my ($fh, $timeout) = @_;
4590
4591     my $sel = new IO::Select;
4592     $sel->add($fh);
4593
4594     my $res = '';
4595     my $buf;
4596
4597     my @ready;
4598     while (scalar (@ready = $sel->can_read($timeout))) {
4599         my $count;
4600         if ($count = $fh->sysread($buf, 8192)) {
4601             if ($buf =~ /^(.*)\(qemu\) $/s) {
4602                 $res .= $1;
4603                 last;
4604             } else {
4605                 $res .= $buf;
4606             }
4607         } else {
4608             if (!defined($count)) {
4609                 die "$!\n";
4610             }
4611             last;
4612         }
4613     }
4614
4615     die "monitor read timeout\n" if !scalar(@ready);
4616
4617     return $res;
4618 }
4619
4620 sub qemu_block_resize {
4621     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4622
4623     my $running = check_running($vmid);
4624
4625     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4626
4627     return if !$running;
4628
4629     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4630
4631 }
4632
4633 sub qemu_volume_snapshot {
4634     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4635
4636     my $running = check_running($vmid);
4637
4638     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4639         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4640     } else {
4641         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4642     }
4643 }
4644
4645 sub qemu_volume_snapshot_delete {
4646     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4647
4648     my $running = check_running($vmid);
4649
4650     if($running) {
4651
4652         $running = undef;
4653         my $conf = PVE::QemuConfig->load_config($vmid);
4654         foreach_drive($conf, sub {
4655             my ($ds, $drive) = @_;
4656             $running = 1 if $drive->{file} eq $volid;
4657         });
4658     }
4659
4660     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4661         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4662     } else {
4663         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4664     }
4665 }
4666
4667 sub set_migration_caps {
4668     my ($vmid) = @_;
4669
4670     my $cap_ref = [];
4671
4672     my $enabled_cap = {
4673         "auto-converge" => 1,
4674         "xbzrle" => 1,
4675         "x-rdma-pin-all" => 0,
4676         "zero-blocks" => 0,
4677         "compress" => 0
4678     };
4679
4680     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4681
4682     for my $supported_capability (@$supported_capabilities) {
4683         push @$cap_ref, {
4684             capability => $supported_capability->{capability},
4685             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4686         };
4687     }
4688
4689     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4690 }
4691
4692 my $fast_plug_option = {
4693     'lock' => 1,
4694     'name' => 1,
4695     'onboot' => 1,
4696     'shares' => 1,
4697     'startup' => 1,
4698     'description' => 1,
4699     'protection' => 1,
4700     'vmstatestorage' => 1,
4701     'hookscript' => 1,
4702     'tags' => 1,
4703 };
4704
4705 # hotplug changes in [PENDING]
4706 # $selection hash can be used to only apply specified options, for
4707 # example: { cores => 1 } (only apply changed 'cores')
4708 # $errors ref is used to return error messages
4709 sub vmconfig_hotplug_pending {
4710     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4711
4712     my $defaults = load_defaults();
4713     my $arch = get_vm_arch($conf);
4714     my $machine_type = get_vm_machine($conf, undef, $arch);
4715
4716     # commit values which do not have any impact on running VM first
4717     # Note: those option cannot raise errors, we we do not care about
4718     # $selection and always apply them.
4719
4720     my $add_error = sub {
4721         my ($opt, $msg) = @_;
4722         $errors->{$opt} = "hotplug problem - $msg";
4723     };
4724
4725     my $changes = 0;
4726     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4727         if ($fast_plug_option->{$opt}) {
4728             $conf->{$opt} = $conf->{pending}->{$opt};
4729             delete $conf->{pending}->{$opt};
4730             $changes = 1;
4731         }
4732     }
4733
4734     if ($changes) {
4735         PVE::QemuConfig->write_config($vmid, $conf);
4736     }
4737
4738     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4739
4740     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4741     foreach my $opt (sort keys %$pending_delete_hash) {
4742         next if $selection && !$selection->{$opt};
4743         my $force = $pending_delete_hash->{$opt}->{force};
4744         eval {
4745             if ($opt eq 'hotplug') {
4746                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4747             } elsif ($opt eq 'tablet') {
4748                 die "skip\n" if !$hotplug_features->{usb};
4749                 if ($defaults->{tablet}) {
4750                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4751                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4752                         if $arch eq 'aarch64';
4753                 } else {
4754                     vm_deviceunplug($vmid, $conf, 'tablet');
4755                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4756                 }
4757             } elsif ($opt =~ m/^usb\d+/) {
4758                 die "skip\n";
4759                 # since we cannot reliably hot unplug usb devices
4760                 # we are disabling it
4761                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4762                 vm_deviceunplug($vmid, $conf, $opt);
4763             } elsif ($opt eq 'vcpus') {
4764                 die "skip\n" if !$hotplug_features->{cpu};
4765                 qemu_cpu_hotplug($vmid, $conf, undef);
4766             } elsif ($opt eq 'balloon') {
4767                 # enable balloon device is not hotpluggable
4768                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4769                 # here we reset the ballooning value to memory
4770                 my $balloon = $conf->{memory} || $defaults->{memory};
4771                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4772             } elsif ($fast_plug_option->{$opt}) {
4773                 # do nothing
4774             } elsif ($opt =~ m/^net(\d+)$/) {
4775                 die "skip\n" if !$hotplug_features->{network};
4776                 vm_deviceunplug($vmid, $conf, $opt);
4777             } elsif (is_valid_drivename($opt)) {
4778                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4779                 vm_deviceunplug($vmid, $conf, $opt);
4780                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4781             } elsif ($opt =~ m/^memory$/) {
4782                 die "skip\n" if !$hotplug_features->{memory};
4783                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4784             } elsif ($opt eq 'cpuunits') {
4785                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4786             } elsif ($opt eq 'cpulimit') {
4787                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4788             } else {
4789                 die "skip\n";
4790             }
4791         };
4792         if (my $err = $@) {
4793             &$add_error($opt, $err) if $err ne "skip\n";
4794         } else {
4795             delete $conf->{$opt};
4796             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4797         }
4798     }
4799
4800     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4801     $apply_pending_cloudinit = sub {
4802         return if $apply_pending_cloudinit_done; # once is enough
4803         $apply_pending_cloudinit_done = 1; # once is enough
4804
4805         my ($key, $value) = @_;
4806
4807         my @cloudinit_opts = keys %$confdesc_cloudinit;
4808         foreach my $opt (keys %{$conf->{pending}}) {
4809             next if !grep { $_ eq $opt } @cloudinit_opts;
4810             $conf->{$opt} = delete $conf->{pending}->{$opt};
4811         }
4812
4813         my $new_conf = { %$conf };
4814         $new_conf->{$key} = $value;
4815         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4816     };
4817
4818     foreach my $opt (keys %{$conf->{pending}}) {
4819         next if $selection && !$selection->{$opt};
4820         my $value = $conf->{pending}->{$opt};
4821         eval {
4822             if ($opt eq 'hotplug') {
4823                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4824             } elsif ($opt eq 'tablet') {
4825                 die "skip\n" if !$hotplug_features->{usb};
4826                 if ($value == 1) {
4827                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4828                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4829                         if $arch eq 'aarch64';
4830                 } elsif ($value == 0) {
4831                     vm_deviceunplug($vmid, $conf, 'tablet');
4832                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4833                 }
4834             } elsif ($opt =~ m/^usb\d+$/) {
4835                 die "skip\n";
4836                 # since we cannot reliably hot unplug usb devices
4837                 # we are disabling it
4838                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4839                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4840                 die "skip\n" if !$d;
4841                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4842             } elsif ($opt eq 'vcpus') {
4843                 die "skip\n" if !$hotplug_features->{cpu};
4844                 qemu_cpu_hotplug($vmid, $conf, $value);
4845             } elsif ($opt eq 'balloon') {
4846                 # enable/disable balloning device is not hotpluggable
4847                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4848                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4849                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4850
4851                 # allow manual ballooning if shares is set to zero
4852                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4853                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4854                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4855                 }
4856             } elsif ($opt =~ m/^net(\d+)$/) {
4857                 # some changes can be done without hotplug
4858                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4859                                     $vmid, $opt, $value, $arch, $machine_type);
4860             } elsif (is_valid_drivename($opt)) {
4861                 die "skip\n" if $opt eq 'efidisk0';
4862                 # some changes can be done without hotplug
4863                 my $drive = parse_drive($opt, $value);
4864                 if (drive_is_cloudinit($drive)) {
4865                     &$apply_pending_cloudinit($opt, $value);
4866                 }
4867                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4868                                      $vmid, $opt, $value, $arch, $machine_type);
4869             } elsif ($opt =~ m/^memory$/) { #dimms
4870                 die "skip\n" if !$hotplug_features->{memory};
4871                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4872             } elsif ($opt eq 'cpuunits') {
4873                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4874             } elsif ($opt eq 'cpulimit') {
4875                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4876                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4877             } else {
4878                 die "skip\n";  # skip non-hot-pluggable options
4879             }
4880         };
4881         if (my $err = $@) {
4882             &$add_error($opt, $err) if $err ne "skip\n";
4883         } else {
4884             $conf->{$opt} = $value;
4885             delete $conf->{pending}->{$opt};
4886         }
4887     }
4888
4889     PVE::QemuConfig->write_config($vmid, $conf);
4890 }
4891
4892 sub try_deallocate_drive {
4893     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4894
4895     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4896         my $volid = $drive->{file};
4897         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4898             my $sid = PVE::Storage::parse_volume_id($volid);
4899             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4900
4901             # check if the disk is really unused
4902             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4903                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4904             PVE::Storage::vdisk_free($storecfg, $volid);
4905             return 1;
4906         } else {
4907             # If vm is not owner of this disk remove from config
4908             return 1;
4909         }
4910     }
4911
4912     return undef;
4913 }
4914
4915 sub vmconfig_delete_or_detach_drive {
4916     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4917
4918     my $drive = parse_drive($opt, $conf->{$opt});
4919
4920     my $rpcenv = PVE::RPCEnvironment::get();
4921     my $authuser = $rpcenv->get_user();
4922
4923     if ($force) {
4924         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4925         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4926     } else {
4927         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4928     }
4929 }
4930
4931
4932
4933 sub vmconfig_apply_pending {
4934     my ($vmid, $conf, $storecfg, $errors) = @_;
4935
4936     my $add_apply_error = sub {
4937         my ($opt, $msg) = @_;
4938         my $err_msg = "unable to apply pending change $opt : $msg";
4939         $errors->{$opt} = $err_msg;
4940         warn $err_msg;
4941     };
4942
4943     # cold plug
4944
4945     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4946     foreach my $opt (sort keys %$pending_delete_hash) {
4947         my $force = $pending_delete_hash->{$opt}->{force};
4948         eval {
4949             if ($opt =~ m/^unused/) {
4950                 die "internal error";
4951             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4952                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4953             }
4954         };
4955         if (my $err = $@) {
4956             $add_apply_error->($opt, $err);
4957         } else {
4958             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4959             delete $conf->{$opt};
4960         }
4961     }
4962
4963     PVE::QemuConfig->cleanup_pending($conf);
4964
4965     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4966         next if $opt eq 'delete'; # just to be sure
4967         eval {
4968             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4969                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4970             }
4971         };
4972         if (my $err = $@) {
4973             $add_apply_error->($opt, $err);
4974         } else {
4975             $conf->{$opt} = delete $conf->{pending}->{$opt};
4976         }
4977     }
4978
4979     # write all changes at once to avoid unnecessary i/o
4980     PVE::QemuConfig->write_config($vmid, $conf);
4981 }
4982
4983 my $safe_num_ne = sub {
4984     my ($a, $b) = @_;
4985
4986     return 0 if !defined($a) && !defined($b);
4987     return 1 if !defined($a);
4988     return 1 if !defined($b);
4989
4990     return $a != $b;
4991 };
4992
4993 my $safe_string_ne = sub {
4994     my ($a, $b) = @_;
4995
4996     return 0 if !defined($a) && !defined($b);
4997     return 1 if !defined($a);
4998     return 1 if !defined($b);
4999
5000     return $a ne $b;
5001 };
5002
5003 sub vmconfig_update_net {
5004     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5005
5006     my $newnet = parse_net($value);
5007
5008     if ($conf->{$opt}) {
5009         my $oldnet = parse_net($conf->{$opt});
5010
5011         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5012             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5013             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5014             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5015
5016             # for non online change, we try to hot-unplug
5017             die "skip\n" if !$hotplug;
5018             vm_deviceunplug($vmid, $conf, $opt);
5019         } else {
5020
5021             die "internal error" if $opt !~ m/net(\d+)/;
5022             my $iface = "tap${vmid}i$1";
5023
5024             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5025                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5026                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5027                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5028                 PVE::Network::tap_unplug($iface);
5029                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5030             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5031                 # Rate can be applied on its own but any change above needs to
5032                 # include the rate in tap_plug since OVS resets everything.
5033                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5034             }
5035
5036             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5037                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5038             }
5039
5040             return 1;
5041         }
5042     }
5043
5044     if ($hotplug) {
5045         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5046     } else {
5047         die "skip\n";
5048     }
5049 }
5050
5051 sub vmconfig_update_disk {
5052     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5053
5054     my $drive = parse_drive($opt, $value);
5055
5056     if ($conf->{$opt}) {
5057
5058         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5059
5060             my $media = $drive->{media} || 'disk';
5061             my $oldmedia = $old_drive->{media} || 'disk';
5062             die "unable to change media type\n" if $media ne $oldmedia;
5063
5064             if (!drive_is_cdrom($old_drive)) {
5065
5066                 if ($drive->{file} ne $old_drive->{file}) {
5067
5068                     die "skip\n" if !$hotplug;
5069
5070                     # unplug and register as unused
5071                     vm_deviceunplug($vmid, $conf, $opt);
5072                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5073
5074                 } else {
5075                     # update existing disk
5076
5077                     # skip non hotpluggable value
5078                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5079                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5080                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5081                         &$safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5082                         &$safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5083                         die "skip\n";
5084                     }
5085
5086                     # apply throttle
5087                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5088                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5089                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5090                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5091                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5092                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5093                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5094                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5095                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5096                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5097                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5098                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5099                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5100                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5101                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5102                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5103                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5104                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5105
5106                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5107                                                    ($drive->{mbps} || 0)*1024*1024,
5108                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5109                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5110                                                    $drive->{iops} || 0,
5111                                                    $drive->{iops_rd} || 0,
5112                                                    $drive->{iops_wr} || 0,
5113                                                    ($drive->{mbps_max} || 0)*1024*1024,
5114                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5115                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5116                                                    $drive->{iops_max} || 0,
5117                                                    $drive->{iops_rd_max} || 0,
5118                                                    $drive->{iops_wr_max} || 0,
5119                                                    $drive->{bps_max_length} || 1,
5120                                                    $drive->{bps_rd_max_length} || 1,
5121                                                    $drive->{bps_wr_max_length} || 1,
5122                                                    $drive->{iops_max_length} || 1,
5123                                                    $drive->{iops_rd_max_length} || 1,
5124                                                    $drive->{iops_wr_max_length} || 1);
5125
5126                     }
5127
5128                     return 1;
5129                 }
5130
5131             } else { # cdrom
5132
5133                 if ($drive->{file} eq 'none') {
5134                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5135                     if (drive_is_cloudinit($old_drive)) {
5136                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5137                     }
5138                 } else {
5139                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5140
5141                     # force eject if locked
5142                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5143
5144                     if ($path) {
5145                         mon_cmd($vmid, "blockdev-change-medium",
5146                             id => "$opt", filename => "$path");
5147                     }
5148                 }
5149
5150                 return 1;
5151             }
5152         }
5153     }
5154
5155     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5156     # hotplug new disks
5157     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5158     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5159 }
5160
5161 sub vm_start {
5162     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5163         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage, $timeout) = @_;
5164
5165     PVE::QemuConfig->lock_config($vmid, sub {
5166         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5167
5168         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5169
5170         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5171
5172         PVE::QemuConfig->check_lock($conf)
5173             if !($skiplock || $is_suspended);
5174
5175         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5176
5177         # clean up leftover reboot request files
5178         eval { clear_reboot_request($vmid); };
5179         warn $@ if $@;
5180
5181         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5182             vmconfig_apply_pending($vmid, $conf, $storecfg);
5183             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5184         }
5185
5186         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5187
5188         my $defaults = load_defaults();
5189
5190         # set environment variable useful inside network script
5191         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5192
5193         my $local_volumes = {};
5194
5195         if ($targetstorage) {
5196             foreach_drive($conf, sub {
5197                 my ($ds, $drive) = @_;
5198
5199                 return if drive_is_cdrom($drive);
5200
5201                 my $volid = $drive->{file};
5202
5203                 return if !$volid;
5204
5205                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5206
5207                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5208                 return if $scfg->{shared};
5209                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5210             });
5211
5212             my $format = undef;
5213
5214             foreach my $opt (sort keys %$local_volumes) {
5215
5216                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5217                 my $drive = parse_drive($opt, $conf->{$opt});
5218
5219                 # If a remote storage is specified and the format of the original
5220                 # volume is not available there, fall back to the default format.
5221                 # Otherwise use the same format as the original.
5222                 if ($targetstorage && $targetstorage ne "1") {
5223                     $storeid = $targetstorage;
5224                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5225                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5226                     my $fileFormat = qemu_img_format($scfg, $volname);
5227                     $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5228                 } else {
5229                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5230                     $format = qemu_img_format($scfg, $volname);
5231                 }
5232
5233                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5234                 my $newdrive = $drive;
5235                 $newdrive->{format} = $format;
5236                 $newdrive->{file} = $newvolid;
5237                 my $drivestr = print_drive($newdrive);
5238                 $local_volumes->{$opt} = $drivestr;
5239                 #pass drive to conf for command line
5240                 $conf->{$opt} = $drivestr;
5241             }
5242         }
5243
5244         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5245
5246         if ($is_suspended) {
5247             # enforce machine type on suspended vm to ensure HW compatibility
5248             $forcemachine = $conf->{runningmachine};
5249             print "Resuming suspended VM\n";
5250         }
5251
5252         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5253
5254         my $migration_ip;
5255         my $get_migration_ip = sub {
5256             my ($cidr, $nodename) = @_;
5257
5258             return $migration_ip if defined($migration_ip);
5259
5260             if (!defined($cidr)) {
5261                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5262                 $cidr = $dc_conf->{migration}->{network};
5263             }
5264
5265             if (defined($cidr)) {
5266                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5267
5268                 die "could not get IP: no address configured on local " .
5269                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5270
5271                 die "could not get IP: multiple addresses configured on local " .
5272                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5273
5274                 $migration_ip = @$ips[0];
5275             }
5276
5277             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5278                 if !defined($migration_ip);
5279
5280             return $migration_ip;
5281         };
5282
5283         my $migrate_uri;
5284         if ($statefile) {
5285             if ($statefile eq 'tcp') {
5286                 my $localip = "localhost";
5287                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5288                 my $nodename = nodename();
5289
5290                 if (!defined($migration_type)) {
5291                     if (defined($datacenterconf->{migration}->{type})) {
5292                         $migration_type = $datacenterconf->{migration}->{type};
5293                     } else {
5294                         $migration_type = 'secure';
5295                     }
5296                 }
5297
5298                 if ($migration_type eq 'insecure') {
5299                     $localip = $get_migration_ip->($migration_network, $nodename);
5300                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5301                 }
5302
5303                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5304                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5305                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5306                 push @$cmd, '-incoming', $migrate_uri;
5307                 push @$cmd, '-S';
5308
5309             } elsif ($statefile eq 'unix') {
5310                 # should be default for secure migrations as a ssh TCP forward
5311                 # tunnel is not deterministic reliable ready and fails regurarly
5312                 # to set up in time, so use UNIX socket forwards
5313                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5314                 unlink $socket_addr;
5315
5316                 $migrate_uri = "unix:$socket_addr";
5317
5318                 push @$cmd, '-incoming', $migrate_uri;
5319                 push @$cmd, '-S';
5320
5321             } elsif (-e $statefile) {
5322                 push @$cmd, '-loadstate', $statefile;
5323             } else {
5324                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5325                 push @$vollist, $statefile;
5326                 push @$cmd, '-loadstate', $statepath;
5327             }
5328         } elsif ($paused) {
5329             push @$cmd, '-S';
5330         }
5331
5332         # host pci devices
5333         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5334           my $d = parse_hostpci($conf->{"hostpci$i"});
5335           next if !$d;
5336           my $pcidevices = $d->{pciid};
5337           foreach my $pcidevice (@$pcidevices) {
5338                 my $pciid = $pcidevice->{id};
5339
5340                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5341                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5342                 die "no pci device info for device '$pciid'\n" if !$info;
5343
5344                 if ($d->{mdev}) {
5345                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5346                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5347                 } else {
5348                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5349                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5350                     die "can't reset pci device '$pciid'\n"
5351                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5352                 }
5353           }
5354         }
5355
5356         PVE::Storage::activate_volumes($storecfg, $vollist);
5357
5358         eval {
5359             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5360                 outfunc => sub {}, errfunc => sub {});
5361         };
5362         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5363         # timeout should be more than enough here...
5364         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5365
5366         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5367                                                   : $defaults->{cpuunits};
5368
5369         my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
5370         my %run_params = (
5371             timeout => $statefile ? undef : $start_timeout,
5372             umask => 0077,
5373             noerr => 1,
5374         );
5375
5376         # when migrating, prefix QEMU output so other side can pick up any
5377         # errors that might occur and show the user
5378         if ($migratedfrom) {
5379             $run_params{quiet} = 1;
5380             $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5381         }
5382
5383         my %properties = (
5384             Slice => 'qemu.slice',
5385             KillMode => 'none',
5386             CPUShares => $cpuunits
5387         );
5388
5389         if (my $cpulimit = $conf->{cpulimit}) {
5390             $properties{CPUQuota} = int($cpulimit * 100);
5391         }
5392         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5393
5394         my $run_qemu = sub {
5395             PVE::Tools::run_fork sub {
5396                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5397
5398                 my $exitcode = run_command($cmd, %run_params);
5399                 die "QEMU exited with code $exitcode\n" if $exitcode;
5400             };
5401         };
5402
5403         if ($conf->{hugepages}) {
5404
5405             my $code = sub {
5406                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5407                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5408
5409                 PVE::QemuServer::Memory::hugepages_mount();
5410                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5411
5412                 eval { $run_qemu->() };
5413                 if (my $err = $@) {
5414                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5415                     die $err;
5416                 }
5417
5418                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5419             };
5420             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5421
5422         } else {
5423             eval { $run_qemu->() };
5424         }
5425
5426         if (my $err = $@) {
5427             # deactivate volumes if start fails
5428             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5429             die "start failed: $err";
5430         }
5431
5432         print "migration listens on $migrate_uri\n" if $migrate_uri;
5433
5434         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5435             eval { mon_cmd($vmid, "cont"); };
5436             warn $@ if $@;
5437         }
5438
5439         #start nbd server for storage migration
5440         if ($targetstorage) {
5441             my $nodename = nodename();
5442             my $localip = $get_migration_ip->($migration_network, $nodename);
5443             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5444             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5445
5446             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5447
5448             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5449
5450             foreach my $opt (sort keys %$local_volumes) {
5451                 my $drivestr = $local_volumes->{$opt};
5452                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5453                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5454                 print "storage migration listens on $migrate_storage_uri volume:$drivestr\n";
5455             }
5456         }
5457
5458         if ($migratedfrom) {
5459             eval {
5460                 set_migration_caps($vmid);
5461             };
5462             warn $@ if $@;
5463
5464             if ($spice_port) {
5465                 print "spice listens on port $spice_port\n";
5466                 if ($spice_ticket) {
5467                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5468                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5469                 }
5470             }
5471
5472         } else {
5473             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5474                 if !$statefile && $conf->{balloon};
5475
5476             foreach my $opt (keys %$conf) {
5477                 next if $opt !~  m/^net\d+$/;
5478                 my $nicconf = parse_net($conf->{$opt});
5479                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5480             }
5481         }
5482
5483         mon_cmd($vmid, 'qom-set',
5484                     path => "machine/peripheral/balloon0",
5485                     property => "guest-stats-polling-interval",
5486                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5487
5488         if ($is_suspended) {
5489             print "Resumed VM, removing state\n";
5490             if (my $vmstate = $conf->{vmstate}) {
5491                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5492                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5493             }
5494             delete $conf->@{qw(lock vmstate runningmachine)};
5495             PVE::QemuConfig->write_config($vmid, $conf);
5496         }
5497
5498         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5499     });
5500 }
5501
5502 sub vm_commandline {
5503     my ($storecfg, $vmid, $snapname) = @_;
5504
5505     my $conf = PVE::QemuConfig->load_config($vmid);
5506     my $forcemachine;
5507
5508     if ($snapname) {
5509         my $snapshot = $conf->{snapshots}->{$snapname};
5510         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5511
5512         # check for a 'runningmachine' in snapshot
5513         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5514
5515         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5516
5517         $conf = $snapshot;
5518     }
5519
5520     my $defaults = load_defaults();
5521
5522     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5523
5524     return PVE::Tools::cmd2string($cmd);
5525 }
5526
5527 sub vm_reset {
5528     my ($vmid, $skiplock) = @_;
5529
5530     PVE::QemuConfig->lock_config($vmid, sub {
5531
5532         my $conf = PVE::QemuConfig->load_config($vmid);
5533
5534         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5535
5536         mon_cmd($vmid, "system_reset");
5537     });
5538 }
5539
5540 sub get_vm_volumes {
5541     my ($conf) = @_;
5542
5543     my $vollist = [];
5544     foreach_volid($conf, sub {
5545         my ($volid, $attr) = @_;
5546
5547         return if $volid =~ m|^/|;
5548
5549         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5550         return if !$sid;
5551
5552         push @$vollist, $volid;
5553     });
5554
5555     return $vollist;
5556 }
5557
5558 sub vm_stop_cleanup {
5559     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5560
5561     eval {
5562
5563         if (!$keepActive) {
5564             my $vollist = get_vm_volumes($conf);
5565             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5566         }
5567
5568         foreach my $ext (qw(mon qmp pid vnc qga)) {
5569             unlink "/var/run/qemu-server/${vmid}.$ext";
5570         }
5571
5572         if ($conf->{ivshmem}) {
5573             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5574             # just delete it for now, VMs which have this already open do not
5575             # are affected, but new VMs will get a separated one. If this
5576             # becomes an issue we either add some sort of ref-counting or just
5577             # add a "don't delete on stop" flag to the ivshmem format.
5578             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5579         }
5580
5581         foreach my $key (keys %$conf) {
5582             next if $key !~ m/^hostpci(\d+)$/;
5583             my $hostpciindex = $1;
5584             my $d = parse_hostpci($conf->{$key});
5585             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5586
5587             foreach my $pci (@{$d->{pciid}}) {
5588                 my $pciid = $pci->{id};
5589                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5590             }
5591         }
5592
5593         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5594     };
5595     warn $@ if $@; # avoid errors - just warn
5596 }
5597
5598 # call only in locked context
5599 sub _do_vm_stop {
5600     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5601
5602     my $pid = check_running($vmid, $nocheck);
5603     return if !$pid;
5604
5605     my $conf;
5606     if (!$nocheck) {
5607         $conf = PVE::QemuConfig->load_config($vmid);
5608         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5609         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5610             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5611             $timeout = $opts->{down} if $opts->{down};
5612         }
5613         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5614     }
5615
5616     eval {
5617         if ($shutdown) {
5618             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5619                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5620             } else {
5621                 mon_cmd($vmid, "system_powerdown");
5622             }
5623         } else {
5624             mon_cmd($vmid, "quit");
5625         }
5626     };
5627     my $err = $@;
5628
5629     if (!$err) {
5630         $timeout = 60 if !defined($timeout);
5631
5632         my $count = 0;
5633         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5634             $count++;
5635             sleep 1;
5636         }
5637
5638         if ($count >= $timeout) {
5639             if ($force) {
5640                 warn "VM still running - terminating now with SIGTERM\n";
5641                 kill 15, $pid;
5642             } else {
5643                 die "VM quit/powerdown failed - got timeout\n";
5644             }
5645         } else {
5646             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5647             return;
5648         }
5649     } else {
5650         if ($force) {
5651             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5652             kill 15, $pid;
5653         } else {
5654             die "VM quit/powerdown failed\n";
5655         }
5656     }
5657
5658     # wait again
5659     $timeout = 10;
5660
5661     my $count = 0;
5662     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5663         $count++;
5664         sleep 1;
5665     }
5666
5667     if ($count >= $timeout) {
5668         warn "VM still running - terminating now with SIGKILL\n";
5669         kill 9, $pid;
5670         sleep 1;
5671     }
5672
5673     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5674 }
5675
5676 # Note: use $nocheck to skip tests if VM configuration file exists.
5677 # We need that when migration VMs to other nodes (files already moved)
5678 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5679 sub vm_stop {
5680     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5681
5682     $force = 1 if !defined($force) && !$shutdown;
5683
5684     if ($migratedfrom){
5685         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5686         kill 15, $pid if $pid;
5687         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5688         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5689         return;
5690     }
5691
5692     PVE::QemuConfig->lock_config($vmid, sub {
5693         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5694    });
5695 }
5696
5697 sub vm_reboot {
5698     my ($vmid, $timeout) = @_;
5699
5700     PVE::QemuConfig->lock_config($vmid, sub {
5701         eval {
5702
5703             # only reboot if running, as qmeventd starts it again on a stop event
5704             return if !check_running($vmid);
5705
5706             create_reboot_request($vmid);
5707
5708             my $storecfg = PVE::Storage::config();
5709             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5710
5711         };
5712         if (my $err = $@) {
5713             # avoid that the next normal shutdown will be confused for a reboot
5714             clear_reboot_request($vmid);
5715             die $err;
5716         }
5717    });
5718 }
5719
5720 # note: if using the statestorage parameter, the caller has to check privileges
5721 sub vm_suspend {
5722     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5723
5724     my $conf;
5725     my $path;
5726     my $storecfg;
5727     my $vmstate;
5728
5729     PVE::QemuConfig->lock_config($vmid, sub {
5730
5731         $conf = PVE::QemuConfig->load_config($vmid);
5732
5733         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5734         PVE::QemuConfig->check_lock($conf)
5735             if !($skiplock || $is_backing_up);
5736
5737         die "cannot suspend to disk during backup\n"
5738             if $is_backing_up && $includestate;
5739
5740         if ($includestate) {
5741             $conf->{lock} = 'suspending';
5742             my $date = strftime("%Y-%m-%d", localtime(time()));
5743             $storecfg = PVE::Storage::config();
5744             if (!$statestorage) {
5745                 $statestorage = find_vmstate_storage($conf, $storecfg);
5746                 # check permissions for the storage
5747                 my $rpcenv = PVE::RPCEnvironment::get();
5748                 if ($rpcenv->{type} ne 'cli') {
5749                     my $authuser = $rpcenv->get_user();
5750                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5751                 }
5752             }
5753
5754
5755             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5756             $path = PVE::Storage::path($storecfg, $vmstate);
5757             PVE::QemuConfig->write_config($vmid, $conf);
5758         } else {
5759             mon_cmd($vmid, "stop");
5760         }
5761     });
5762
5763     if ($includestate) {
5764         # save vm state
5765         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5766
5767         eval {
5768             mon_cmd($vmid, "savevm-start", statefile => $path);
5769             for(;;) {
5770                 my $state = mon_cmd($vmid, "query-savevm");
5771                 if (!$state->{status}) {
5772                     die "savevm not active\n";
5773                 } elsif ($state->{status} eq 'active') {
5774                     sleep(1);
5775                     next;
5776                 } elsif ($state->{status} eq 'completed') {
5777                     print "State saved, quitting\n";
5778                     last;
5779                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5780                     die "query-savevm failed with error '$state->{error}'\n"
5781                 } else {
5782                     die "query-savevm returned status '$state->{status}'\n";
5783                 }
5784             }
5785         };
5786         my $err = $@;
5787
5788         PVE::QemuConfig->lock_config($vmid, sub {
5789             $conf = PVE::QemuConfig->load_config($vmid);
5790             if ($err) {
5791                 # cleanup, but leave suspending lock, to indicate something went wrong
5792                 eval {
5793                     mon_cmd($vmid, "savevm-end");
5794                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5795                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5796                     delete $conf->@{qw(vmstate runningmachine)};
5797                     PVE::QemuConfig->write_config($vmid, $conf);
5798                 };
5799                 warn $@ if $@;
5800                 die $err;
5801             }
5802
5803             die "lock changed unexpectedly\n"
5804                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5805
5806             mon_cmd($vmid, "quit");
5807             $conf->{lock} = 'suspended';
5808             PVE::QemuConfig->write_config($vmid, $conf);
5809         });
5810     }
5811 }
5812
5813 sub vm_resume {
5814     my ($vmid, $skiplock, $nocheck) = @_;
5815
5816     PVE::QemuConfig->lock_config($vmid, sub {
5817         my $res = mon_cmd($vmid, 'query-status');
5818         my $resume_cmd = 'cont';
5819
5820         if ($res->{status} && $res->{status} eq 'suspended') {
5821             $resume_cmd = 'system_wakeup';
5822         }
5823
5824         if (!$nocheck) {
5825
5826             my $conf = PVE::QemuConfig->load_config($vmid);
5827
5828             PVE::QemuConfig->check_lock($conf)
5829                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5830         }
5831
5832         mon_cmd($vmid, $resume_cmd);
5833     });
5834 }
5835
5836 sub vm_sendkey {
5837     my ($vmid, $skiplock, $key) = @_;
5838
5839     PVE::QemuConfig->lock_config($vmid, sub {
5840
5841         my $conf = PVE::QemuConfig->load_config($vmid);
5842
5843         # there is no qmp command, so we use the human monitor command
5844         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5845         die $res if $res ne '';
5846     });
5847 }
5848
5849 # vzdump restore implementaion
5850
5851 sub tar_archive_read_firstfile {
5852     my $archive = shift;
5853
5854     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5855
5856     # try to detect archive type first
5857     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5858         die "unable to open file '$archive'\n";
5859     my $firstfile = <$fh>;
5860     kill 15, $pid;
5861     close $fh;
5862
5863     die "ERROR: archive contaions no data\n" if !$firstfile;
5864     chomp $firstfile;
5865
5866     return $firstfile;
5867 }
5868
5869 sub tar_restore_cleanup {
5870     my ($storecfg, $statfile) = @_;
5871
5872     print STDERR "starting cleanup\n";
5873
5874     if (my $fd = IO::File->new($statfile, "r")) {
5875         while (defined(my $line = <$fd>)) {
5876             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5877                 my $volid = $2;
5878                 eval {
5879                     if ($volid =~ m|^/|) {
5880                         unlink $volid || die 'unlink failed\n';
5881                     } else {
5882                         PVE::Storage::vdisk_free($storecfg, $volid);
5883                     }
5884                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5885                 };
5886                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5887             } else {
5888                 print STDERR "unable to parse line in statfile - $line";
5889             }
5890         }
5891         $fd->close();
5892     }
5893 }
5894
5895 sub restore_archive {
5896     my ($archive, $vmid, $user, $opts) = @_;
5897
5898     my $format = $opts->{format};
5899     my $comp;
5900
5901     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5902         $format = 'tar' if !$format;
5903         $comp = 'gzip';
5904     } elsif ($archive =~ m/\.tar$/) {
5905         $format = 'tar' if !$format;
5906     } elsif ($archive =~ m/.tar.lzo$/) {
5907         $format = 'tar' if !$format;
5908         $comp = 'lzop';
5909     } elsif ($archive =~ m/\.vma$/) {
5910         $format = 'vma' if !$format;
5911     } elsif ($archive =~ m/\.vma\.gz$/) {
5912         $format = 'vma' if !$format;
5913         $comp = 'gzip';
5914     } elsif ($archive =~ m/\.vma\.lzo$/) {
5915         $format = 'vma' if !$format;
5916         $comp = 'lzop';
5917     } else {
5918         $format = 'vma' if !$format; # default
5919     }
5920
5921     # try to detect archive format
5922     if ($format eq 'tar') {
5923         return restore_tar_archive($archive, $vmid, $user, $opts);
5924     } else {
5925         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5926     }
5927 }
5928
5929 sub restore_update_config_line {
5930     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5931
5932     return if $line =~ m/^\#qmdump\#/;
5933     return if $line =~ m/^\#vzdump\#/;
5934     return if $line =~ m/^lock:/;
5935     return if $line =~ m/^unused\d+:/;
5936     return if $line =~ m/^parent:/;
5937
5938     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5939     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5940         # try to convert old 1.X settings
5941         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5942         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5943             my ($model, $macaddr) = split(/\=/, $devconfig);
5944             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5945             my $net = {
5946                 model => $model,
5947                 bridge => "vmbr$ind",
5948                 macaddr => $macaddr,
5949             };
5950             my $netstr = print_net($net);
5951
5952             print $outfd "net$cookie->{netcount}: $netstr\n";
5953             $cookie->{netcount}++;
5954         }
5955     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5956         my ($id, $netstr) = ($1, $2);
5957         my $net = parse_net($netstr);
5958         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5959         $netstr = print_net($net);
5960         print $outfd "$id: $netstr\n";
5961     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5962         my $virtdev = $1;
5963         my $value = $3;
5964         my $di = parse_drive($virtdev, $value);
5965         if (defined($di->{backup}) && !$di->{backup}) {
5966             print $outfd "#$line";
5967         } elsif ($map->{$virtdev}) {
5968             delete $di->{format}; # format can change on restore
5969             $di->{file} = $map->{$virtdev};
5970             $value = print_drive($di);
5971             print $outfd "$virtdev: $value\n";
5972         } else {
5973             print $outfd $line;
5974         }
5975     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5976         my $vmgenid = $1;
5977         if ($vmgenid ne '0') {
5978             # always generate a new vmgenid if there was a valid one setup
5979             $vmgenid = generate_uuid();
5980         }
5981         print $outfd "vmgenid: $vmgenid\n";
5982     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5983         my ($uuid, $uuid_str);
5984         UUID::generate($uuid);
5985         UUID::unparse($uuid, $uuid_str);
5986         my $smbios1 = parse_smbios1($2);
5987         $smbios1->{uuid} = $uuid_str;
5988         print $outfd $1.print_smbios1($smbios1)."\n";
5989     } else {
5990         print $outfd $line;
5991     }
5992 }
5993
5994 sub scan_volids {
5995     my ($cfg, $vmid) = @_;
5996
5997     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5998
5999     my $volid_hash = {};
6000     foreach my $storeid (keys %$info) {
6001         foreach my $item (@{$info->{$storeid}}) {
6002             next if !($item->{volid} && $item->{size});
6003             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6004             $volid_hash->{$item->{volid}} = $item;
6005         }
6006     }
6007
6008     return $volid_hash;
6009 }
6010
6011 sub is_volume_in_use {
6012     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6013
6014     my $path = PVE::Storage::path($storecfg, $volid);
6015
6016     my $scan_config = sub {
6017         my ($cref, $snapname) = @_;
6018
6019         foreach my $key (keys %$cref) {
6020             my $value = $cref->{$key};
6021             if (is_valid_drivename($key)) {
6022                 next if $skip_drive && $key eq $skip_drive;
6023                 my $drive = parse_drive($key, $value);
6024                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6025                 return 1 if $volid eq $drive->{file};
6026                 if ($drive->{file} =~ m!^/!) {
6027                     return 1 if $drive->{file} eq $path;
6028                 } else {
6029                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6030                     next if !$storeid;
6031                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6032                     next if !$scfg;
6033                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6034                 }
6035             }
6036         }
6037
6038         return 0;
6039     };
6040
6041     return 1 if &$scan_config($conf);
6042
6043     undef $skip_drive;
6044
6045     foreach my $snapname (keys %{$conf->{snapshots}}) {
6046         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6047     }
6048
6049     return 0;
6050 }
6051
6052 sub update_disksize {
6053     my ($drive, $volid_hash) = @_;
6054
6055     my $volid = $drive->{file};
6056     return undef if !defined($volid);
6057
6058     my $oldsize = $drive->{size};
6059     my $newsize = $volid_hash->{$volid}->{size};
6060
6061     if (defined($newsize) && defined($oldsize) && $newsize != $oldsize) {
6062         $drive->{size} = $newsize;
6063
6064         my $old_fmt = PVE::JSONSchema::format_size($oldsize);
6065         my $new_fmt = PVE::JSONSchema::format_size($newsize);
6066
6067         return wantarray ? ($drive, $old_fmt, $new_fmt) : $drive;
6068     }
6069
6070     return undef;
6071 }
6072
6073 sub update_disk_config {
6074     my ($vmid, $conf, $volid_hash) = @_;
6075
6076     my $changes;
6077     my $prefix = "VM $vmid:";
6078
6079     # used and unused disks
6080     my $referenced = {};
6081
6082     # Note: it is allowed to define multiple storages with same path (alias), so
6083     # we need to check both 'volid' and real 'path' (two different volid can point
6084     # to the same path).
6085
6086     my $referencedpath = {};
6087
6088     # update size info
6089     foreach my $opt (keys %$conf) {
6090         if (is_valid_drivename($opt)) {
6091             my $drive = parse_drive($opt, $conf->{$opt});
6092             my $volid = $drive->{file};
6093             next if !$volid;
6094
6095             # mark volid as "in-use" for next step
6096             $referenced->{$volid} = 1;
6097             if ($volid_hash->{$volid} &&
6098                 (my $path = $volid_hash->{$volid}->{path})) {
6099                 $referencedpath->{$path} = 1;
6100             }
6101
6102             next if drive_is_cdrom($drive);
6103             next if !$volid_hash->{$volid};
6104
6105             my ($updated, $old_size, $new_size) = update_disksize($drive, $volid_hash);
6106             if (defined($updated)) {
6107                 $changes = 1;
6108                 $conf->{$opt} = print_drive($updated);
6109                 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
6110             }
6111         }
6112     }
6113
6114     # remove 'unusedX' entry if volume is used
6115     foreach my $opt (keys %$conf) {
6116         next if $opt !~ m/^unused\d+$/;
6117         my $volid = $conf->{$opt};
6118         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6119         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6120             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6121             $changes = 1;
6122             delete $conf->{$opt};
6123         }
6124
6125         $referenced->{$volid} = 1;
6126         $referencedpath->{$path} = 1 if $path;
6127     }
6128
6129     foreach my $volid (sort keys %$volid_hash) {
6130         next if $volid =~ m/vm-$vmid-state-/;
6131         next if $referenced->{$volid};
6132         my $path = $volid_hash->{$volid}->{path};
6133         next if !$path; # just to be sure
6134         next if $referencedpath->{$path};
6135         $changes = 1;
6136         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6137         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6138         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6139     }
6140
6141     return $changes;
6142 }
6143
6144 sub rescan {
6145     my ($vmid, $nolock, $dryrun) = @_;
6146
6147     my $cfg = PVE::Storage::config();
6148
6149     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6150     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6151     foreach my $stor (keys %{$cfg->{ids}}) {
6152         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6153     }
6154
6155     print "rescan volumes...\n";
6156     my $volid_hash = scan_volids($cfg, $vmid);
6157
6158     my $updatefn =  sub {
6159         my ($vmid) = @_;
6160
6161         my $conf = PVE::QemuConfig->load_config($vmid);
6162
6163         PVE::QemuConfig->check_lock($conf);
6164
6165         my $vm_volids = {};
6166         foreach my $volid (keys %$volid_hash) {
6167             my $info = $volid_hash->{$volid};
6168             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6169         }
6170
6171         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6172
6173         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6174     };
6175
6176     if (defined($vmid)) {
6177         if ($nolock) {
6178             &$updatefn($vmid);
6179         } else {
6180             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6181         }
6182     } else {
6183         my $vmlist = config_list();
6184         foreach my $vmid (keys %$vmlist) {
6185             if ($nolock) {
6186                 &$updatefn($vmid);
6187             } else {
6188                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6189             }
6190         }
6191     }
6192 }
6193
6194 sub restore_vma_archive {
6195     my ($archive, $vmid, $user, $opts, $comp) = @_;
6196
6197     my $readfrom = $archive;
6198
6199     my $cfg = PVE::Storage::config();
6200     my $commands = [];
6201     my $bwlimit = $opts->{bwlimit};
6202
6203     my $dbg_cmdstring = '';
6204     my $add_pipe = sub {
6205         my ($cmd) = @_;
6206         push @$commands, $cmd;
6207         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6208         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6209         $readfrom = '-';
6210     };
6211
6212     my $input = undef;
6213     if ($archive eq '-') {
6214         $input = '<&STDIN';
6215     } else {
6216         # If we use a backup from a PVE defined storage we also consider that
6217         # storage's rate limit:
6218         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6219         if (defined($volid)) {
6220             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6221             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6222             if ($readlimit) {
6223                 print STDERR "applying read rate limit: $readlimit\n";
6224                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6225                 $add_pipe->($cstream);
6226             }
6227         }
6228     }
6229
6230     if ($comp) {
6231         my $cmd;
6232         if ($comp eq 'gzip') {
6233             $cmd = ['zcat', $readfrom];
6234         } elsif ($comp eq 'lzop') {
6235             $cmd = ['lzop', '-d', '-c', $readfrom];
6236         } else {
6237             die "unknown compression method '$comp'\n";
6238         }
6239         $add_pipe->($cmd);
6240     }
6241
6242     my $tmpdir = "/var/tmp/vzdumptmp$$";
6243     rmtree $tmpdir;
6244
6245     # disable interrupts (always do cleanups)
6246     local $SIG{INT} =
6247         local $SIG{TERM} =
6248         local $SIG{QUIT} =
6249         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6250
6251     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6252     POSIX::mkfifo($mapfifo, 0600);
6253     my $fifofh;
6254
6255     my $openfifo = sub {
6256         open($fifofh, '>', $mapfifo) || die $!;
6257     };
6258
6259     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6260
6261     my $oldtimeout;
6262     my $timeout = 5;
6263
6264     my $devinfo = {};
6265
6266     my $rpcenv = PVE::RPCEnvironment::get();
6267
6268     my $conffile = PVE::QemuConfig->config_file($vmid);
6269     my $tmpfn = "$conffile.$$.tmp";
6270
6271     # Note: $oldconf is undef if VM does not exist
6272     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6273     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6274
6275     my %storage_limits;
6276
6277     my $print_devmap = sub {
6278         my $virtdev_hash = {};
6279
6280         my $cfgfn = "$tmpdir/qemu-server.conf";
6281
6282         # we can read the config - that is already extracted
6283         my $fh = IO::File->new($cfgfn, "r") ||
6284             "unable to read qemu-server.conf - $!\n";
6285
6286         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6287         if (-f $fwcfgfn) {
6288             my $pve_firewall_dir = '/etc/pve/firewall';
6289             mkdir $pve_firewall_dir; # make sure the dir exists
6290             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6291         }
6292
6293         while (defined(my $line = <$fh>)) {
6294             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6295                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6296                 die "archive does not contain data for drive '$virtdev'\n"
6297                     if !$devinfo->{$devname};
6298                 if (defined($opts->{storage})) {
6299                     $storeid = $opts->{storage} || 'local';
6300                 } elsif (!$storeid) {
6301                     $storeid = 'local';
6302                 }
6303                 $format = 'raw' if !$format;
6304                 $devinfo->{$devname}->{devname} = $devname;
6305                 $devinfo->{$devname}->{virtdev} = $virtdev;
6306                 $devinfo->{$devname}->{format} = $format;
6307                 $devinfo->{$devname}->{storeid} = $storeid;
6308
6309                 # check permission on storage
6310                 my $pool = $opts->{pool}; # todo: do we need that?
6311                 if ($user ne 'root@pam') {
6312                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6313                 }
6314
6315                 $storage_limits{$storeid} = $bwlimit;
6316
6317                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6318             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6319                 my $virtdev = $1;
6320                 my $drive = parse_drive($virtdev, $2);
6321                 if (drive_is_cloudinit($drive)) {
6322                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6323                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6324                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6325
6326                     my $d = {
6327                         format => $format,
6328                         storeid => $opts->{storage} // $storeid,
6329                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6330                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6331                         name => "vm-$vmid-cloudinit",
6332                         is_cloudinit => 1,
6333                     };
6334                     $virtdev_hash->{$virtdev} = $d;
6335                 }
6336             }
6337         }
6338
6339         foreach my $key (keys %storage_limits) {
6340             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6341             next if !$limit;
6342             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6343             $storage_limits{$key} = $limit * 1024;
6344         }
6345
6346         foreach my $devname (keys %$devinfo) {
6347             die "found no device mapping information for device '$devname'\n"
6348                 if !$devinfo->{$devname}->{virtdev};
6349         }
6350
6351         # create empty/temp config
6352         if ($oldconf) {
6353             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6354             foreach_drive($oldconf, sub {
6355                 my ($ds, $drive) = @_;
6356
6357                 return if drive_is_cdrom($drive, 1);
6358
6359                 my $volid = $drive->{file};
6360                 return if !$volid || $volid =~ m|^/|;
6361
6362                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6363                 return if !$path || !$owner || ($owner != $vmid);
6364
6365                 # Note: only delete disk we want to restore
6366                 # other volumes will become unused
6367                 if ($virtdev_hash->{$ds}) {
6368                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6369                     if (my $err = $@) {
6370                         warn $err;
6371                     }
6372                 }
6373             });
6374
6375             # delete vmstate files, after the restore we have no snapshots anymore
6376             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6377                 my $snap = $oldconf->{snapshots}->{$snapname};
6378                 if ($snap->{vmstate}) {
6379                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6380                     if (my $err = $@) {
6381                         warn $err;
6382                     }
6383                 }
6384             }
6385         }
6386
6387         my $map = {};
6388         foreach my $virtdev (sort keys %$virtdev_hash) {
6389             my $d = $virtdev_hash->{$virtdev};
6390             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6391             my $storeid = $d->{storeid};
6392             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6393
6394             my $map_opts = '';
6395             if (my $limit = $storage_limits{$storeid}) {
6396                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6397             }
6398
6399             # test if requested format is supported
6400             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6401             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6402             $d->{format} = $defFormat if !$supported;
6403
6404             my $name;
6405             if ($d->{is_cloudinit}) {
6406                 $name = $d->{name};
6407                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6408             }
6409
6410             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6411             print STDERR "new volume ID is '$volid'\n";
6412             $d->{volid} = $volid;
6413
6414             PVE::Storage::activate_volumes($cfg, [$volid]);
6415
6416             my $write_zeros = 1;
6417             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6418                 $write_zeros = 0;
6419             }
6420
6421             if (!$d->{is_cloudinit}) {
6422                 my $path = PVE::Storage::path($cfg, $volid);
6423
6424                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6425
6426                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6427             }
6428             $map->{$virtdev} = $volid;
6429         }
6430
6431         $fh->seek(0, 0) || die "seek failed - $!\n";
6432
6433         my $outfd = new IO::File ($tmpfn, "w") ||
6434             die "unable to write config for VM $vmid\n";
6435
6436         my $cookie = { netcount => 0 };
6437         while (defined(my $line = <$fh>)) {
6438             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6439         }
6440
6441         $fh->close();
6442         $outfd->close();
6443     };
6444
6445     eval {
6446         # enable interrupts
6447         local $SIG{INT} =
6448             local $SIG{TERM} =
6449             local $SIG{QUIT} =
6450             local $SIG{HUP} =
6451             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6452         local $SIG{ALRM} = sub { die "got timeout\n"; };
6453
6454         $oldtimeout = alarm($timeout);
6455
6456         my $parser = sub {
6457             my $line = shift;
6458
6459             print "$line\n";
6460
6461             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6462                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6463                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6464             } elsif ($line =~ m/^CTIME: /) {
6465                 # we correctly received the vma config, so we can disable
6466                 # the timeout now for disk allocation (set to 10 minutes, so
6467                 # that we always timeout if something goes wrong)
6468                 alarm(600);
6469                 &$print_devmap();
6470                 print $fifofh "done\n";
6471                 my $tmp = $oldtimeout || 0;
6472                 $oldtimeout = undef;
6473                 alarm($tmp);
6474                 close($fifofh);
6475             }
6476         };
6477
6478         print "restore vma archive: $dbg_cmdstring\n";
6479         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6480     };
6481     my $err = $@;
6482
6483     alarm($oldtimeout) if $oldtimeout;
6484
6485     my $vollist = [];
6486     foreach my $devname (keys %$devinfo) {
6487         my $volid = $devinfo->{$devname}->{volid};
6488         push @$vollist, $volid if $volid;
6489     }
6490
6491     PVE::Storage::deactivate_volumes($cfg, $vollist);
6492
6493     unlink $mapfifo;
6494
6495     if ($err) {
6496         rmtree $tmpdir;
6497         unlink $tmpfn;
6498
6499         foreach my $devname (keys %$devinfo) {
6500             my $volid = $devinfo->{$devname}->{volid};
6501             next if !$volid;
6502             eval {
6503                 if ($volid =~ m|^/|) {
6504                     unlink $volid || die 'unlink failed\n';
6505                 } else {
6506                     PVE::Storage::vdisk_free($cfg, $volid);
6507                 }
6508                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6509             };
6510             print STDERR "unable to cleanup '$volid' - $@" if $@;
6511         }
6512         die $err;
6513     }
6514
6515     rmtree $tmpdir;
6516
6517     rename($tmpfn, $conffile) ||
6518         die "unable to commit configuration file '$conffile'\n";
6519
6520     PVE::Cluster::cfs_update(); # make sure we read new file
6521
6522     eval { rescan($vmid, 1); };
6523     warn $@ if $@;
6524 }
6525
6526 sub restore_tar_archive {
6527     my ($archive, $vmid, $user, $opts) = @_;
6528
6529     if ($archive ne '-') {
6530         my $firstfile = tar_archive_read_firstfile($archive);
6531         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6532             if $firstfile ne 'qemu-server.conf';
6533     }
6534
6535     my $storecfg = PVE::Storage::config();
6536
6537     # avoid zombie disks when restoring over an existing VM -> cleanup first
6538     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6539     # skiplock=1 because qmrestore has set the 'create' lock itself already
6540     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6541     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6542
6543     my $tocmd = "/usr/lib/qemu-server/qmextract";
6544
6545     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6546     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6547     $tocmd .= ' --prealloc' if $opts->{prealloc};
6548     $tocmd .= ' --info' if $opts->{info};
6549
6550     # tar option "xf" does not autodetect compression when read from STDIN,
6551     # so we pipe to zcat
6552     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6553         PVE::Tools::shellquote("--to-command=$tocmd");
6554
6555     my $tmpdir = "/var/tmp/vzdumptmp$$";
6556     mkpath $tmpdir;
6557
6558     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6559     local $ENV{VZDUMP_VMID} = $vmid;
6560     local $ENV{VZDUMP_USER} = $user;
6561
6562     my $conffile = PVE::QemuConfig->config_file($vmid);
6563     my $tmpfn = "$conffile.$$.tmp";
6564
6565     # disable interrupts (always do cleanups)
6566     local $SIG{INT} =
6567         local $SIG{TERM} =
6568         local $SIG{QUIT} =
6569         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6570
6571     eval {
6572         # enable interrupts
6573         local $SIG{INT} =
6574             local $SIG{TERM} =
6575             local $SIG{QUIT} =
6576             local $SIG{HUP} =
6577             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6578
6579         if ($archive eq '-') {
6580             print "extracting archive from STDIN\n";
6581             run_command($cmd, input => "<&STDIN");
6582         } else {
6583             print "extracting archive '$archive'\n";
6584             run_command($cmd);
6585         }
6586
6587         return if $opts->{info};
6588
6589         # read new mapping
6590         my $map = {};
6591         my $statfile = "$tmpdir/qmrestore.stat";
6592         if (my $fd = IO::File->new($statfile, "r")) {
6593             while (defined (my $line = <$fd>)) {
6594                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6595                     $map->{$1} = $2 if $1;
6596                 } else {
6597                     print STDERR "unable to parse line in statfile - $line\n";
6598                 }
6599             }
6600             $fd->close();
6601         }
6602
6603         my $confsrc = "$tmpdir/qemu-server.conf";
6604
6605         my $srcfd = new IO::File($confsrc, "r") ||
6606             die "unable to open file '$confsrc'\n";
6607
6608         my $outfd = new IO::File ($tmpfn, "w") ||
6609             die "unable to write config for VM $vmid\n";
6610
6611         my $cookie = { netcount => 0 };
6612         while (defined (my $line = <$srcfd>)) {
6613             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6614         }
6615
6616         $srcfd->close();
6617         $outfd->close();
6618     };
6619     if (my $err = $@) {
6620         unlink $tmpfn;
6621         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6622         die $err;
6623     }
6624
6625     rmtree $tmpdir;
6626
6627     rename $tmpfn, $conffile ||
6628         die "unable to commit configuration file '$conffile'\n";
6629
6630     PVE::Cluster::cfs_update(); # make sure we read new file
6631
6632     eval { rescan($vmid, 1); };
6633     warn $@ if $@;
6634 };
6635
6636 sub foreach_storage_used_by_vm {
6637     my ($conf, $func) = @_;
6638
6639     my $sidhash = {};
6640
6641     foreach_drive($conf, sub {
6642         my ($ds, $drive) = @_;
6643         return if drive_is_cdrom($drive);
6644
6645         my $volid = $drive->{file};
6646
6647         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6648         $sidhash->{$sid} = $sid if $sid;
6649     });
6650
6651     foreach my $sid (sort keys %$sidhash) {
6652         &$func($sid);
6653     }
6654 }
6655
6656 my $qemu_snap_storage = {
6657     rbd => 1,
6658 };
6659 sub do_snapshots_with_qemu {
6660     my ($storecfg, $volid) = @_;
6661
6662     my $storage_name = PVE::Storage::parse_volume_id($volid);
6663     my $scfg = $storecfg->{ids}->{$storage_name};
6664
6665     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6666         return 1;
6667     }
6668
6669     if ($volid =~ m/\.(qcow2|qed)$/){
6670         return 1;
6671     }
6672
6673     return undef;
6674 }
6675
6676 sub qga_check_running {
6677     my ($vmid, $nowarn) = @_;
6678
6679     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6680     if ($@) {
6681         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6682         return 0;
6683     }
6684     return 1;
6685 }
6686
6687 sub template_create {
6688     my ($vmid, $conf, $disk) = @_;
6689
6690     my $storecfg = PVE::Storage::config();
6691
6692     foreach_drive($conf, sub {
6693         my ($ds, $drive) = @_;
6694
6695         return if drive_is_cdrom($drive);
6696         return if $disk && $ds ne $disk;
6697
6698         my $volid = $drive->{file};
6699         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6700
6701         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6702         $drive->{file} = $voliddst;
6703         $conf->{$ds} = print_drive($drive);
6704         PVE::QemuConfig->write_config($vmid, $conf);
6705     });
6706 }
6707
6708 sub convert_iscsi_path {
6709     my ($path) = @_;
6710
6711     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6712         my $portal = $1;
6713         my $target = $2;
6714         my $lun = $3;
6715
6716         my $initiator_name = get_initiator_name();
6717
6718         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6719                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6720     }
6721
6722     die "cannot convert iscsi path '$path', unkown format\n";
6723 }
6724
6725 sub qemu_img_convert {
6726     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6727
6728     my $storecfg = PVE::Storage::config();
6729     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6730     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6731
6732     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6733
6734     my $cachemode;
6735     my $src_path;
6736     my $src_is_iscsi = 0;
6737     my $src_format;
6738
6739     if ($src_storeid) {
6740         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6741         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6742         $src_format = qemu_img_format($src_scfg, $src_volname);
6743         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6744         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6745         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6746     } elsif (-f $src_volid) {
6747         $src_path = $src_volid;
6748         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6749             $src_format = $1;
6750         }
6751     }
6752
6753     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6754
6755     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6756     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6757     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6758     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6759
6760     my $cmd = [];
6761     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6762     push @$cmd, '-l', "snapshot.name=$snapname"
6763         if $snapname && $src_format && $src_format eq "qcow2";
6764     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6765     push @$cmd, '-T', $cachemode if defined($cachemode);
6766
6767     if ($src_is_iscsi) {
6768         push @$cmd, '--image-opts';
6769         $src_path = convert_iscsi_path($src_path);
6770     } elsif ($src_format) {
6771         push @$cmd, '-f', $src_format;
6772     }
6773
6774     if ($dst_is_iscsi) {
6775         push @$cmd, '--target-image-opts';
6776         $dst_path = convert_iscsi_path($dst_path);
6777     } else {
6778         push @$cmd, '-O', $dst_format;
6779     }
6780
6781     push @$cmd, $src_path;
6782
6783     if (!$dst_is_iscsi && $is_zero_initialized) {
6784         push @$cmd, "zeroinit:$dst_path";
6785     } else {
6786         push @$cmd, $dst_path;
6787     }
6788
6789     my $parser = sub {
6790         my $line = shift;
6791         if($line =~ m/\((\S+)\/100\%\)/){
6792             my $percent = $1;
6793             my $transferred = int($size * $percent / 100);
6794             my $remaining = $size - $transferred;
6795
6796             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6797         }
6798
6799     };
6800
6801     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6802     my $err = $@;
6803     die "copy failed: $err" if $err;
6804 }
6805
6806 sub qemu_img_format {
6807     my ($scfg, $volname) = @_;
6808
6809     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6810         return $1;
6811     } else {
6812         return "raw";
6813     }
6814 }
6815
6816 sub qemu_drive_mirror {
6817     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6818
6819     $jobs = {} if !$jobs;
6820
6821     my $qemu_target;
6822     my $format;
6823     $jobs->{"drive-$drive"} = {};
6824
6825     if ($dst_volid =~ /^nbd:/) {
6826         $qemu_target = $dst_volid;
6827         $format = "nbd";
6828     } else {
6829         my $storecfg = PVE::Storage::config();
6830         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6831
6832         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6833
6834         $format = qemu_img_format($dst_scfg, $dst_volname);
6835
6836         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6837
6838         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6839     }
6840
6841     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6842     $opts->{format} = $format if $format;
6843
6844     if (defined($bwlimit)) {
6845         $opts->{speed} = $bwlimit * 1024;
6846         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6847     } else {
6848         print "drive mirror is starting for drive-$drive\n";
6849     }
6850
6851     # if a job already runs for this device we get an error, catch it for cleanup
6852     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6853     if (my $err = $@) {
6854         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6855         warn "$@\n" if $@;
6856         die "mirroring error: $err\n";
6857     }
6858
6859     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6860 }
6861
6862 sub qemu_drive_mirror_monitor {
6863     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6864
6865     eval {
6866         my $err_complete = 0;
6867
6868         while (1) {
6869             die "storage migration timed out\n" if $err_complete > 300;
6870
6871             my $stats = mon_cmd($vmid, "query-block-jobs");
6872
6873             my $running_mirror_jobs = {};
6874             foreach my $stat (@$stats) {
6875                 next if $stat->{type} ne 'mirror';
6876                 $running_mirror_jobs->{$stat->{device}} = $stat;
6877             }
6878
6879             my $readycounter = 0;
6880
6881             foreach my $job (keys %$jobs) {
6882
6883                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6884                     print "$job : finished\n";
6885                     delete $jobs->{$job};
6886                     next;
6887                 }
6888
6889                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6890
6891                 my $busy = $running_mirror_jobs->{$job}->{busy};
6892                 my $ready = $running_mirror_jobs->{$job}->{ready};
6893                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6894                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6895                     my $remaining = $total - $transferred;
6896                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6897
6898                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6899                 }
6900
6901                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6902             }
6903
6904             last if scalar(keys %$jobs) == 0;
6905
6906             if ($readycounter == scalar(keys %$jobs)) {
6907                 print "all mirroring jobs are ready \n";
6908                 last if $skipcomplete; #do the complete later
6909
6910                 if ($vmiddst && $vmiddst != $vmid) {
6911                     my $agent_running = $qga && qga_check_running($vmid);
6912                     if ($agent_running) {
6913                         print "freeze filesystem\n";
6914                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6915                     } else {
6916                         print "suspend vm\n";
6917                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6918                     }
6919
6920                     # if we clone a disk for a new target vm, we don't switch the disk
6921                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6922
6923                     if ($agent_running) {
6924                         print "unfreeze filesystem\n";
6925                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6926                     } else {
6927                         print "resume vm\n";
6928                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6929                     }
6930
6931                     last;
6932                 } else {
6933
6934                     foreach my $job (keys %$jobs) {
6935                         # try to switch the disk if source and destination are on the same guest
6936                         print "$job: Completing block job...\n";
6937
6938                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6939                         if ($@ =~ m/cannot be completed/) {
6940                             print "$job: Block job cannot be completed, try again.\n";
6941                             $err_complete++;
6942                         }else {
6943                             print "$job: Completed successfully.\n";
6944                             $jobs->{$job}->{complete} = 1;
6945                         }
6946                     }
6947                 }
6948             }
6949             sleep 1;
6950         }
6951     };
6952     my $err = $@;
6953
6954     if ($err) {
6955         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6956         die "mirroring error: $err";
6957     }
6958
6959 }
6960
6961 sub qemu_blockjobs_cancel {
6962     my ($vmid, $jobs) = @_;
6963
6964     foreach my $job (keys %$jobs) {
6965         print "$job: Cancelling block job\n";
6966         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6967         $jobs->{$job}->{cancel} = 1;
6968     }
6969
6970     while (1) {
6971         my $stats = mon_cmd($vmid, "query-block-jobs");
6972
6973         my $running_jobs = {};
6974         foreach my $stat (@$stats) {
6975             $running_jobs->{$stat->{device}} = $stat;
6976         }
6977
6978         foreach my $job (keys %$jobs) {
6979
6980             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6981                 print "$job: Done.\n";
6982                 delete $jobs->{$job};
6983             }
6984         }
6985
6986         last if scalar(keys %$jobs) == 0;
6987
6988         sleep 1;
6989     }
6990 }
6991
6992 sub clone_disk {
6993     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6994         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6995
6996     my $newvolid;
6997
6998     if (!$full) {
6999         print "create linked clone of drive $drivename ($drive->{file})\n";
7000         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7001         push @$newvollist, $newvolid;
7002     } else {
7003
7004         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7005         $storeid = $storage if $storage;
7006
7007         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7008         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7009
7010         print "create full clone of drive $drivename ($drive->{file})\n";
7011         my $name = undef;
7012         if (drive_is_cloudinit($drive)) {
7013             $name = "vm-$newvmid-cloudinit";
7014             $name .= ".$dst_format" if $dst_format ne 'raw';
7015             $snapname = undef;
7016             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7017         }
7018         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7019         push @$newvollist, $newvolid;
7020
7021         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7022
7023         if (drive_is_cloudinit($drive)) {
7024             goto no_data_clone;
7025         }
7026
7027         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7028         if (!$running || $snapname) {
7029             # TODO: handle bwlimits
7030             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7031         } else {
7032
7033             my $kvmver = get_running_qemu_version ($vmid);
7034             if (!min_version($kvmver, 2, 7)) {
7035                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7036                     if $drive->{iothread};
7037             }
7038
7039             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7040         }
7041     }
7042
7043 no_data_clone:
7044     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7045
7046     my $disk = $drive;
7047     $disk->{format} = undef;
7048     $disk->{file} = $newvolid;
7049     $disk->{size} = $size;
7050
7051     return $disk;
7052 }
7053
7054 sub get_running_qemu_version {
7055     my ($vmid) = @_;
7056     my $res = mon_cmd($vmid, "query-version");
7057     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7058 }
7059
7060 sub qemu_use_old_bios_files {
7061     my ($machine_type) = @_;
7062
7063     return if !$machine_type;
7064
7065     my $use_old_bios_files = undef;
7066
7067     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7068         $machine_type = $1;
7069         $use_old_bios_files = 1;
7070     } else {
7071         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7072         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7073         # load new efi bios files on migration. So this hack is required to allow
7074         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7075         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7076         $use_old_bios_files = !min_version($version, 2, 4);
7077     }
7078
7079     return ($use_old_bios_files, $machine_type);
7080 }
7081
7082 sub create_efidisk($$$$$) {
7083     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7084
7085     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7086     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7087
7088     my $vars_size_b = -s $ovmf_vars;
7089     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7090     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7091     PVE::Storage::activate_volumes($storecfg, [$volid]);
7092
7093     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7094     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7095
7096     return ($volid, $size/1024);
7097 }
7098
7099 sub vm_iothreads_list {
7100     my ($vmid) = @_;
7101
7102     my $res = mon_cmd($vmid, 'query-iothreads');
7103
7104     my $iothreads = {};
7105     foreach my $iothread (@$res) {
7106         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7107     }
7108
7109     return $iothreads;
7110 }
7111
7112 sub scsihw_infos {
7113     my ($conf, $drive) = @_;
7114
7115     my $maxdev = 0;
7116
7117     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7118         $maxdev = 7;
7119     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7120         $maxdev = 1;
7121     } else {
7122         $maxdev = 256;
7123     }
7124
7125     my $controller = int($drive->{index} / $maxdev);
7126     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7127
7128     return ($maxdev, $controller, $controller_prefix);
7129 }
7130
7131 sub windows_version {
7132     my ($ostype) = @_;
7133
7134     return 0 if !$ostype;
7135
7136     my $winversion = 0;
7137
7138     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7139         $winversion = 5;
7140     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7141         $winversion = 6;
7142     } elsif ($ostype =~ m/^win(\d+)$/) {
7143         $winversion = $1;
7144     }
7145
7146     return $winversion;
7147 }
7148
7149 sub resolve_dst_disk_format {
7150         my ($storecfg, $storeid, $src_volname, $format) = @_;
7151         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7152
7153         if (!$format) {
7154             # if no target format is specified, use the source disk format as hint
7155             if ($src_volname) {
7156                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7157                 $format = qemu_img_format($scfg, $src_volname);
7158             } else {
7159                 return $defFormat;
7160             }
7161         }
7162
7163         # test if requested format is supported - else use default
7164         my $supported = grep { $_ eq $format } @$validFormats;
7165         $format = $defFormat if !$supported;
7166         return $format;
7167 }
7168
7169 sub resolve_first_disk {
7170     my $conf = shift;
7171     my @disks = PVE::QemuServer::valid_drive_names();
7172     my $firstdisk;
7173     foreach my $ds (reverse @disks) {
7174         next if !$conf->{$ds};
7175         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7176         next if PVE::QemuServer::drive_is_cdrom($disk);
7177         $firstdisk = $ds;
7178     }
7179     return $firstdisk;
7180 }
7181
7182 # NOTE: if this logic changes, please update docs & possibly gui logic
7183 sub find_vmstate_storage {
7184     my ($conf, $storecfg) = @_;
7185
7186     # first, return storage from conf if set
7187     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7188
7189     my ($target, $shared, $local);
7190
7191     foreach_storage_used_by_vm($conf, sub {
7192         my ($sid) = @_;
7193         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7194         my $dst = $scfg->{shared} ? \$shared : \$local;
7195         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7196     });
7197
7198     # second, use shared storage where VM has at least one disk
7199     # third, use local storage where VM has at least one disk
7200     # fall back to local storage
7201     $target = $shared // $local // 'local';
7202
7203     return $target;
7204 }
7205
7206 sub generate_uuid {
7207     my ($uuid, $uuid_str);
7208     UUID::generate($uuid);
7209     UUID::unparse($uuid, $uuid_str);
7210     return $uuid_str;
7211 }
7212
7213 sub generate_smbios1_uuid {
7214     return "uuid=".generate_uuid();
7215 }
7216
7217 sub nbd_stop {
7218     my ($vmid) = @_;
7219
7220     mon_cmd($vmid, 'nbd-server-stop');
7221 }
7222
7223 sub create_reboot_request {
7224     my ($vmid) = @_;
7225     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7226         or die "failed to create reboot trigger file: $!\n";
7227     close($fh);
7228 }
7229
7230 sub clear_reboot_request {
7231     my ($vmid) = @_;
7232     my $path = "/run/qemu-server/$vmid.reboot";
7233     my $res = 0;
7234
7235     $res = unlink($path);
7236     die "could not remove reboot request for $vmid: $!"
7237         if !$res && $! != POSIX::ENOENT;
7238
7239     return $res;
7240 }
7241
7242 # bash completion helper
7243
7244 sub complete_backup_archives {
7245     my ($cmdname, $pname, $cvalue) = @_;
7246
7247     my $cfg = PVE::Storage::config();
7248
7249     my $storeid;
7250
7251     if ($cvalue =~ m/^([^:]+):/) {
7252         $storeid = $1;
7253     }
7254
7255     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7256
7257     my $res = [];
7258     foreach my $id (keys %$data) {
7259         foreach my $item (@{$data->{$id}}) {
7260             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7261             push @$res, $item->{volid} if defined($item->{volid});
7262         }
7263     }
7264
7265     return $res;
7266 }
7267
7268 my $complete_vmid_full = sub {
7269     my ($running) = @_;
7270
7271     my $idlist = vmstatus();
7272
7273     my $res = [];
7274
7275     foreach my $id (keys %$idlist) {
7276         my $d = $idlist->{$id};
7277         if (defined($running)) {
7278             next if $d->{template};
7279             next if $running && $d->{status} ne 'running';
7280             next if !$running && $d->{status} eq 'running';
7281         }
7282         push @$res, $id;
7283
7284     }
7285     return $res;
7286 };
7287
7288 sub complete_vmid {
7289     return &$complete_vmid_full();
7290 }
7291
7292 sub complete_vmid_stopped {
7293     return &$complete_vmid_full(0);
7294 }
7295
7296 sub complete_vmid_running {
7297     return &$complete_vmid_full(1);
7298 }
7299
7300 sub complete_storage {
7301
7302     my $cfg = PVE::Storage::config();
7303     my $ids = $cfg->{ids};
7304
7305     my $res = [];
7306     foreach my $sid (keys %$ids) {
7307         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7308         next if !$ids->{$sid}->{content}->{images};
7309         push @$res, $sid;
7310     }
7311
7312     return $res;
7313 }
7314
7315 sub complete_migration_storage {
7316     my ($cmd, $param, $current_value, $all_args) = @_;
7317
7318     my $targetnode = @$all_args[1];
7319
7320     my $cfg = PVE::Storage::config();
7321     my $ids = $cfg->{ids};
7322
7323     my $res = [];
7324     foreach my $sid (keys %$ids) {
7325         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7326         next if !$ids->{$sid}->{content}->{images};
7327         push @$res, $sid;
7328     }
7329
7330     return $res;
7331 }
7332
7333 1;