PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 my $cpu_vendor_list = {
 114     # Intel CPUs
 115     486 => 'GenuineIntel',
 116     pentium => 'GenuineIntel',
 117     pentium2  => 'GenuineIntel',
 118     pentium3  => 'GenuineIntel',
 119     coreduo => 'GenuineIntel',
 120     core2duo => 'GenuineIntel',
 121     Conroe  => 'GenuineIntel',
 122     Penryn  => 'GenuineIntel',
 123     Nehalem  => 'GenuineIntel',
 124     'Nehalem-IBRS'  => 'GenuineIntel',
 125     Westmere => 'GenuineIntel',
 126     'Westmere-IBRS' => 'GenuineIntel',
 127     SandyBridge => 'GenuineIntel',
 128     'SandyBridge-IBRS' => 'GenuineIntel',
 129     IvyBridge => 'GenuineIntel',
 130     'IvyBridge-IBRS' => 'GenuineIntel',
 131     Haswell => 'GenuineIntel',
 132     'Haswell-IBRS' => 'GenuineIntel',
 133     'Haswell-noTSX' => 'GenuineIntel',
 134     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 135     Broadwell => 'GenuineIntel',
 136     'Broadwell-IBRS' => 'GenuineIntel',
 137     'Broadwell-noTSX' => 'GenuineIntel',
 138     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 139     'Skylake-Client' => 'GenuineIntel',
 140     'Skylake-Client-IBRS' => 'GenuineIntel',
 141     'Skylake-Server' => 'GenuineIntel',
 142     'Skylake-Server-IBRS' => 'GenuineIntel',
 143     'Cascadelake-Server' => 'GenuineIntel',
 144     KnightsMill => 'GenuineIntel',
 145
 146
 147     # AMD CPUs
 148     athlon => 'AuthenticAMD',
 149     phenom  => 'AuthenticAMD',
 150     Opteron_G1  => 'AuthenticAMD',
 151     Opteron_G2  => 'AuthenticAMD',
 152     Opteron_G3  => 'AuthenticAMD',
 153     Opteron_G4  => 'AuthenticAMD',
 154     Opteron_G5  => 'AuthenticAMD',
 155     EPYC => 'AuthenticAMD',
 156     'EPYC-IBPB' => 'AuthenticAMD',
 157
 158     # generic types, use vendor from host node
 159     host => 'default',
 160     kvm32 => 'default',
 161     kvm64 => 'default',
 162     qemu32 => 'default',
 163     qemu64 => 'default',
 164     max => 'default',
 165 };
 166
 167 my @supported_cpu_flags = (
 168     'pcid',
 169     'spec-ctrl',
 170     'ibpb',
 171     'ssbd',
 172     'virt-ssbd',
 173     'amd-ssbd',
 174     'amd-no-ssb',
 175     'pdpe1gb',
 176     'md-clear',
 177     'hv-tlbflush',
 178     'hv-evmcs',
 179     'aes'
 180 );
 181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 182
 183 my $cpu_fmt = {
 184     cputype => {
 185         description => "Emulated CPU type.",
 186         type => 'string',
 187         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 188         default => 'kvm64',
 189         default_key => 1,
 190     },
 191     hidden => {
 192         description => "Do not identify as a KVM virtual machine.",
 193         type => 'boolean',
 194         optional => 1,
 195         default => 0
 196     },
 197     'hv-vendor-id' => {
 198         type => 'string',
 199         pattern => qr/[a-zA-Z0-9]{1,12}/,
 200         format_description => 'vendor-id',
 201         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 202         optional => 1,
 203     },
 204     flags => {
 205         description => "List of additional CPU flags separated by ';'."
 206                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 207                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 208         format_description => '+FLAG[;-FLAG...]',
 209         type => 'string',
 210         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 211         optional => 1,
 212     },
 213 };
 214
 215 my $watchdog_fmt = {
 216     model => {
 217         default_key => 1,
 218         type => 'string',
 219         enum => [qw(i6300esb ib700)],
 220         description => "Watchdog type to emulate.",
 221         default => 'i6300esb',
 222         optional => 1,
 223     },
 224     action => {
 225         type => 'string',
 226         enum => [qw(reset shutdown poweroff pause debug none)],
 227         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 228         optional => 1,
 229     },
 230 };
 231 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 232
 233 my $agent_fmt = {
 234     enabled => {
 235         description => "Enable/disable Qemu GuestAgent.",
 236         type => 'boolean',
 237         default => 0,
 238         default_key => 1,
 239     },
 240     fstrim_cloned_disks => {
 241         description => "Run fstrim after cloning/moving a disk.",
 242         type => 'boolean',
 243         optional => 1,
 244         default => 0
 245     },
 246     type => {
 247         description => "Select the agent type",
 248         type => 'string',
 249         default => 'virtio',
 250         optional => 1,
 251         enum => [qw(virtio isa)],
 252     },
 253 };
 254
 255 my $vga_fmt = {
 256     type => {
 257         description => "Select the VGA type.",
 258         type => 'string',
 259         default => 'std',
 260         optional => 1,
 261         default_key => 1,
 262         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 263     },
 264     memory => {
 265         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 266         type => 'integer',
 267         optional => 1,
 268         minimum => 4,
 269         maximum => 512,
 270     },
 271 };
 272
 273 my $ivshmem_fmt = {
 274     size => {
 275         type => 'integer',
 276         minimum => 1,
 277         description => "The size of the file in MB.",
 278     },
 279     name => {
 280         type => 'string',
 281         pattern => '[a-zA-Z0-9\-]+',
 282         optional => 1,
 283         format_description => 'string',
 284         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 285     },
 286 };
 287
 288 my $audio_fmt = {
 289     device => {
 290         type => 'string',
 291         enum => [qw(ich9-intel-hda intel-hda AC97)],
 292         description =>  "Configure an audio device."
 293     },
 294     driver =>  {
 295         type => 'string',
 296         enum => ['spice'],
 297         default => 'spice',
 298         optional => 1,
 299         description => "Driver backend for the audio device."
 300     },
 301 };
 302
 303 my $spice_enhancements_fmt = {
 304     foldersharing => {
 305         type => 'boolean',
 306         optional => 1,
 307         default => '0',
 308         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 309     },
 310     videostreaming =>  {
 311         type => 'string',
 312         enum => ['off', 'all', 'filter'],
 313         default => 'off',
 314         optional => 1,
 315         description => "Enable video streaming. Uses compression for detected video streams."
 316     },
 317 };
 318
 319 my $confdesc = {
 320     onboot => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Specifies whether a VM will be started during system bootup.",
 324         default => 0,
 325     },
 326     autostart => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Automatic restart after crash (currently ignored).",
 330         default => 0,
 331     },
 332     hotplug => {
 333         optional => 1,
 334         type => 'string', format => 'pve-hotplug-features',
 335         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 336         default => 'network,disk,usb',
 337     },
 338     reboot => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 342         default => 1,
 343     },
 344     lock => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Lock/unlock the VM.",
 348         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 349     },
 350     cpulimit => {
 351         optional => 1,
 352         type => 'number',
 353         description => "Limit of CPU usage.",
 354         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 355         minimum => 0,
 356         maximum => 128,
 357         default => 0,
 358     },
 359     cpuunits => {
 360         optional => 1,
 361         type => 'integer',
 362         description => "CPU weight for a VM.",
 363         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 364         minimum => 2,
 365         maximum => 262144,
 366         default => 1024,
 367     },
 368     memory => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 372         minimum => 16,
 373         default => 512,
 374     },
 375     balloon => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 379         minimum => 0,
 380     },
 381     shares => {
 382         optional => 1,
 383         type => 'integer',
 384         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 385         minimum => 0,
 386         maximum => 50000,
 387         default => 1000,
 388     },
 389     keyboard => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 393                        "It should not be necessary to set it.",
 394         enum => PVE::Tools::kvmkeymaplist(),
 395         default => undef,
 396     },
 397     name => {
 398         optional => 1,
 399         type => 'string', format => 'dns-name',
 400         description => "Set a name for the VM. Only used on the configuration web interface.",
 401     },
 402     scsihw => {
 403         optional => 1,
 404         type => 'string',
 405         description => "SCSI controller model",
 406         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 407         default => 'lsi',
 408     },
 409     description => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 413     },
 414     ostype => {
 415         optional => 1,
 416         type => 'string',
 417         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 418         description => "Specify guest operating system.",
 419         verbose_description => <<EODESC,
 420 Specify guest operating system. This is used to enable special
 421 optimization/features for specific operating systems:
 422
 423 [horizontal]
 424 other;; unspecified OS
 425 wxp;; Microsoft Windows XP
 426 w2k;; Microsoft Windows 2000
 427 w2k3;; Microsoft Windows 2003
 428 w2k8;; Microsoft Windows 2008
 429 wvista;; Microsoft Windows Vista
 430 win7;; Microsoft Windows 7
 431 win8;; Microsoft Windows 8/2012/2012r2
 432 win10;; Microsoft Windows 10/2016
 433 l24;; Linux 2.4 Kernel
 434 l26;; Linux 2.6 - 5.X Kernel
 435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 436 EODESC
 437     },
 438     boot => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 442         pattern => '[acdn]{1,4}',
 443         default => 'cdn',
 444     },
 445     bootdisk => {
 446         optional => 1,
 447         type => 'string', format => 'pve-qm-bootdisk',
 448         description => "Enable booting from specified disk.",
 449         pattern => '(ide|sata|scsi|virtio)\d+',
 450     },
 451     smp => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of CPUs. Please use option -sockets instead.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     sockets => {
 459         optional => 1,
 460         type => 'integer',
 461         description => "The number of CPU sockets.",
 462         minimum => 1,
 463         default => 1,
 464     },
 465     cores => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "The number of cores per socket.",
 469         minimum => 1,
 470         default => 1,
 471     },
 472     numa => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable NUMA.",
 476         default => 0,
 477     },
 478     hugepages => {
 479         optional => 1,
 480         type => 'string',
 481         description => "Enable/disable hugepages memory.",
 482         enum => [qw(any 2 1024)],
 483     },
 484     vcpus => {
 485         optional => 1,
 486         type => 'integer',
 487         description => "Number of hotplugged vcpus.",
 488         minimum => 1,
 489         default => 0,
 490     },
 491     acpi => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable ACPI.",
 495         default => 1,
 496     },
 497     agent => {
 498         optional => 1,
 499         description => "Enable/disable Qemu GuestAgent and its properties.",
 500         type => 'string',
 501         format => $agent_fmt,
 502     },
 503     kvm => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable KVM hardware virtualization.",
 507         default => 1,
 508     },
 509     tdf => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Enable/disable time drift fix.",
 513         default => 0,
 514     },
 515     localtime => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 519     },
 520     freeze => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 524     },
 525     vga => {
 526         optional => 1,
 527         type => 'string', format => $vga_fmt,
 528         description => "Configure the VGA hardware.",
 529         verbose_description => "Configure the VGA Hardware. If you want to use ".
 530             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 531             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 532             "is 'std' for all OS types besides some Windows versions (XP and " .
 533             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 534             "display server. For win* OS you can select how many independent " .
 535             "displays you want, Linux guests can add displays them self.\n".
 536             "You can also run without any graphic card, using a serial device as terminal.",
 537     },
 538     watchdog => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-watchdog',
 541         description => "Create a virtual hardware watchdog device.",
 542         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 543             " (by a guest action), the watchdog must be periodically polled " .
 544             "by an agent inside the guest or else the watchdog will reset " .
 545             "the guest (or execute the respective action specified)",
 546     },
 547     startdate => {
 548         optional => 1,
 549         type => 'string',
 550         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 551         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 552         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 553         default => 'now',
 554     },
 555     startup =>  get_standard_option('pve-startup-order'),
 556     template => {
 557         optional => 1,
 558         type => 'boolean',
 559         description => "Enable/disable Template.",
 560         default => 0,
 561     },
 562     args => {
 563         optional => 1,
 564         type => 'string',
 565         description => "Arbitrary arguments passed to kvm.",
 566         verbose_description => <<EODESCR,
 567 Arbitrary arguments passed to kvm, for example:
 568
 569 args: -no-reboot -no-hpet
 570
 571 NOTE: this option is for experts only.
 572 EODESCR
 573     },
 574     tablet => {
 575         optional => 1,
 576         type => 'boolean',
 577         default => 1,
 578         description => "Enable/disable the USB tablet device.",
 579         verbose_description => "Enable/disable the USB tablet device. This device is " .
 580             "usually needed to allow absolute mouse positioning with VNC. " .
 581             "Else the mouse runs out of sync with normal VNC clients. " .
 582             "If you're running lots of console-only guests on one host, " .
 583             "you may consider disabling this to save some context switches. " .
 584             "This is turned off by default if you use spice (-vga=qxl).",
 585     },
 586     migrate_speed => {
 587         optional => 1,
 588         type => 'integer',
 589         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 590         minimum => 0,
 591         default => 0,
 592     },
 593     migrate_downtime => {
 594         optional => 1,
 595         type => 'number',
 596         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 597         minimum => 0,
 598         default => 0.1,
 599     },
 600     cdrom => {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-ide',
 603         typetext => '<volume>',
 604         description => "This is an alias for option -ide2",
 605     },
 606     cpu => {
 607         optional => 1,
 608         description => "Emulated CPU type.",
 609         type => 'string',
 610         format => $cpu_fmt,
 611     },
 612     parent => get_standard_option('pve-snapshot-name', {
 613         optional => 1,
 614         description => "Parent snapshot name. This is used internally, and should not be modified.",
 615     }),
 616     snaptime => {
 617         optional => 1,
 618         description => "Timestamp for snapshots.",
 619         type => 'integer',
 620         minimum => 0,
 621     },
 622     vmstate => {
 623         optional => 1,
 624         type => 'string', format => 'pve-volume-id',
 625         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 626     },
 627     vmstatestorage => get_standard_option('pve-storage-id', {
 628         description => "Default storage for VM state volumes/files.",
 629         optional => 1,
 630     }),
 631     runningmachine => get_standard_option('pve-qemu-machine', {
 632         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 633     }),
 634     machine => get_standard_option('pve-qemu-machine'),
 635     arch => {
 636         description => "Virtual processor architecture. Defaults to the host.",
 637         optional => 1,
 638         type => 'string',
 639         enum => [qw(x86_64 aarch64)],
 640     },
 641     smbios1 => {
 642         description => "Specify SMBIOS type 1 fields.",
 643         type => 'string', format => 'pve-qm-smbios1',
 644         maxLength => 512,
 645         optional => 1,
 646     },
 647     protection => {
 648         optional => 1,
 649         type => 'boolean',
 650         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 651         default => 0,
 652     },
 653     bios => {
 654         optional => 1,
 655         type => 'string',
 656         enum => [ qw(seabios ovmf) ],
 657         description => "Select BIOS implementation.",
 658         default => 'seabios',
 659     },
 660     vmgenid => {
 661         type => 'string',
 662         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 663         format_description => 'UUID',
 664         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 665         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 666             " 128-bit integer value identifier to the guest OS. This allows to".
 667             " notify the guest operating system when the virtual machine is".
 668             " executed with a different configuration (e.g. snapshot execution".
 669             " or creation from a template). The guest operating system notices".
 670             " the change, and is then able to react as appropriate by marking".
 671             " its copies of distributed databases as dirty, re-initializing its".
 672             " random number generator, etc.\n".
 673             "Note that auto-creation only works when done throug API/CLI create".
 674             " or update methods, but not when manually editing the config file.",
 675         default => "1 (autogenerated)",
 676         optional => 1,
 677     },
 678     hookscript => {
 679         type => 'string',
 680         format => 'pve-volume-id',
 681         optional => 1,
 682         description => "Script that will be executed during various steps in the vms lifetime.",
 683     },
 684     ivshmem => {
 685         type => 'string',
 686         format => $ivshmem_fmt,
 687         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 688         optional => 1,
 689     },
 690     audio0 => {
 691         type => 'string',
 692         format => $audio_fmt,
 693         description => "Configure a audio device, useful in combination with QXL/Spice.",
 694         optional => 1
 695     },
 696     spice_enhancements => {
 697         type => 'string',
 698         format => $spice_enhancements_fmt,
 699         description => "Configure additional enhancements for SPICE.",
 700         optional => 1
 701     },
 702     tags => {
 703         type => 'string', format => 'pve-tag-list',
 704         description => 'Tags of the VM. This is only meta information.',
 705         optional => 1,
 706     },
 707 };
 708
 709 my $cicustom_fmt = {
 710     meta => {
 711         type => 'string',
 712         optional => 1,
 713         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 714         format => 'pve-volume-id',
 715         format_description => 'volume',
 716     },
 717     network => {
 718         type => 'string',
 719         optional => 1,
 720         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 721         format => 'pve-volume-id',
 722         format_description => 'volume',
 723     },
 724     user => {
 725         type => 'string',
 726         optional => 1,
 727         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731 };
 732 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 733
 734 my $confdesc_cloudinit = {
 735     citype => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 739         enum => ['configdrive2', 'nocloud'],
 740     },
 741     ciuser => {
 742         optional => 1,
 743         type => 'string',
 744         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 750     },
 751     cicustom => {
 752         optional => 1,
 753         type => 'string',
 754         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 755         format => 'pve-qm-cicustom',
 756     },
 757     searchdomain => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 761     },
 762     nameserver => {
 763         optional => 1,
 764         type => 'string', format => 'address-list',
 765         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 766     },
 767     sshkeys => {
 768         optional => 1,
 769         type => 'string',
 770         format => 'urlencoded',
 771         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 772     },
 773 };
 774
 775 # what about other qemu settings ?
 776 #cpu => 'string',
 777 #machine => 'string',
 778 #fda => 'file',
 779 #fdb => 'file',
 780 #mtdblock => 'file',
 781 #sd => 'file',
 782 #pflash => 'file',
 783 #snapshot => 'bool',
 784 #bootp => 'file',
 785 ##tftp => 'dir',
 786 ##smb => 'dir',
 787 #kernel => 'file',
 788 #append => 'string',
 789 #initrd => 'file',
 790 ##soundhw => 'string',
 791
 792 while (my ($k, $v) = each %$confdesc) {
 793     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 794 }
 795
 796 my $MAX_IDE_DISKS = 4;
 797 my $MAX_SCSI_DISKS = 14;
 798 my $MAX_VIRTIO_DISKS = 16;
 799 my $MAX_SATA_DISKS = 6;
 800 my $MAX_USB_DEVICES = 5;
 801 my $MAX_NETS = 32;
 802 my $MAX_UNUSED_DISKS = 256;
 803 my $MAX_HOSTPCI_DEVICES = 16;
 804 my $MAX_SERIAL_PORTS = 4;
 805 my $MAX_PARALLEL_PORTS = 3;
 806 my $MAX_NUMA = 8;
 807
 808 my $numa_fmt = {
 809     cpus => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "CPUs accessing this NUMA node.",
 813         format_description => "id[-id];...",
 814     },
 815     memory => {
 816         type => "number",
 817         description => "Amount of memory this NUMA node provides.",
 818         optional => 1,
 819     },
 820     hostnodes => {
 821         type => "string",
 822         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 823         description => "Host NUMA nodes to use.",
 824         format_description => "id[-id];...",
 825         optional => 1,
 826     },
 827     policy => {
 828         type => 'string',
 829         enum => [qw(preferred bind interleave)],
 830         description => "NUMA allocation policy.",
 831         optional => 1,
 832     },
 833 };
 834 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 835 my $numadesc = {
 836     optional => 1,
 837     type => 'string', format => $numa_fmt,
 838     description => "NUMA topology.",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 841
 842 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 843     $confdesc->{"numa$i"} = $numadesc;
 844 }
 845
 846 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 847                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 848                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 849 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 850
 851 my $net_fmt_bridge_descr = <<__EOD__;
 852 Bridge to attach the network device to. The Proxmox VE standard bridge
 853 is called 'vmbr0'.
 854
 855 If you do not specify a bridge, we create a kvm user (NATed) network
 856 device, which provides DHCP and DNS services. The following addresses
 857 are used:
 858
 859  10.0.2.2   Gateway
 860  10.0.2.3   DNS Server
 861  10.0.2.4   SMB Server
 862
 863 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 864 __EOD__
 865
 866 my $net_fmt = {
 867     macaddr  => get_standard_option('mac-addr', {
 868         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 869     }),
 870     model => {
 871         type => 'string',
 872         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 873         enum => $nic_model_list,
 874         default_key => 1,
 875     },
 876     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 877     bridge => {
 878         type => 'string',
 879         description => $net_fmt_bridge_descr,
 880         format_description => 'bridge',
 881         optional => 1,
 882     },
 883     queues => {
 884         type => 'integer',
 885         minimum => 0, maximum => 16,
 886         description => 'Number of packet queues to be used on the device.',
 887         optional => 1,
 888     },
 889     rate => {
 890         type => 'number',
 891         minimum => 0,
 892         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 893         optional => 1,
 894     },
 895     tag => {
 896         type => 'integer',
 897         minimum => 1, maximum => 4094,
 898         description => 'VLAN tag to apply to packets on this interface.',
 899         optional => 1,
 900     },
 901     trunks => {
 902         type => 'string',
 903         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 904         description => 'VLAN trunks to pass through this interface.',
 905         format_description => 'vlanid[;vlanid...]',
 906         optional => 1,
 907     },
 908     firewall => {
 909         type => 'boolean',
 910         description => 'Whether this interface should be protected by the firewall.',
 911         optional => 1,
 912     },
 913     link_down => {
 914         type => 'boolean',
 915         description => 'Whether this interface should be disconnected (like pulling the plug).',
 916         optional => 1,
 917     },
 918 };
 919
 920 my $netdesc = {
 921     optional => 1,
 922     type => 'string', format => $net_fmt,
 923     description => "Specify network devices.",
 924 };
 925
 926 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 927
 928 my $ipconfig_fmt = {
 929     ip => {
 930         type => 'string',
 931         format => 'pve-ipv4-config',
 932         format_description => 'IPv4Format/CIDR',
 933         description => 'IPv4 address in CIDR format.',
 934         optional => 1,
 935         default => 'dhcp',
 936     },
 937     gw => {
 938         type => 'string',
 939         format => 'ipv4',
 940         format_description => 'GatewayIPv4',
 941         description => 'Default gateway for IPv4 traffic.',
 942         optional => 1,
 943         requires => 'ip',
 944     },
 945     ip6 => {
 946         type => 'string',
 947         format => 'pve-ipv6-config',
 948         format_description => 'IPv6Format/CIDR',
 949         description => 'IPv6 address in CIDR format.',
 950         optional => 1,
 951         default => 'dhcp',
 952     },
 953     gw6 => {
 954         type => 'string',
 955         format => 'ipv6',
 956         format_description => 'GatewayIPv6',
 957         description => 'Default gateway for IPv6 traffic.',
 958         optional => 1,
 959         requires => 'ip6',
 960     },
 961 };
 962 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 963 my $ipconfigdesc = {
 964     optional => 1,
 965     type => 'string', format => 'pve-qm-ipconfig',
 966     description => <<'EODESCR',
 967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 968
 969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 970
 971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return undef if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $drivename_hash;
1006
1007 my %drivedesc_base = (
1008     volume => { alias => 'file' },
1009     file => {
1010         type => 'string',
1011         format => 'pve-volume-id-or-qm-path',
1012         default_key => 1,
1013         format_description => 'volume',
1014         description => "The drive's backing volume.",
1015     },
1016     media => {
1017         type => 'string',
1018         enum => [qw(cdrom disk)],
1019         description => "The drive's media type.",
1020         default => 'disk',
1021         optional => 1
1022     },
1023     cyls => {
1024         type => 'integer',
1025         description => "Force the drive's physical geometry to have a specific cylinder count.",
1026         optional => 1
1027     },
1028     heads => {
1029         type => 'integer',
1030         description => "Force the drive's physical geometry to have a specific head count.",
1031         optional => 1
1032     },
1033     secs => {
1034         type => 'integer',
1035         description => "Force the drive's physical geometry to have a specific sector count.",
1036         optional => 1
1037     },
1038     trans => {
1039         type => 'string',
1040         enum => [qw(none lba auto)],
1041         description => "Force disk geometry bios translation mode.",
1042         optional => 1,
1043     },
1044     snapshot => {
1045         type => 'boolean',
1046         description => "Controls qemu's snapshot mode feature."
1047             . " If activated, changes made to the disk are temporary and will"
1048             . " be discarded when the VM is shutdown.",
1049         optional => 1,
1050     },
1051     cache => {
1052         type => 'string',
1053         enum => [qw(none writethrough writeback unsafe directsync)],
1054         description => "The drive's cache mode",
1055         optional => 1,
1056     },
1057     format => get_standard_option('pve-qm-image-format'),
1058     size => {
1059         type => 'string',
1060         format => 'disk-size',
1061         format_description => 'DiskSize',
1062         description => "Disk size. This is purely informational and has no effect.",
1063         optional => 1,
1064     },
1065     backup => {
1066         type => 'boolean',
1067         description => "Whether the drive should be included when making backups.",
1068         optional => 1,
1069     },
1070     replicate => {
1071         type => 'boolean',
1072         description => 'Whether the drive should considered for replication jobs.',
1073         optional => 1,
1074         default => 1,
1075     },
1076     rerror => {
1077         type => 'string',
1078         enum => [qw(ignore report stop)],
1079         description => 'Read error action.',
1080         optional => 1,
1081     },
1082     werror => {
1083         type => 'string',
1084         enum => [qw(enospc ignore report stop)],
1085         description => 'Write error action.',
1086         optional => 1,
1087     },
1088     aio => {
1089         type => 'string',
1090         enum => [qw(native threads)],
1091         description => 'AIO type to use.',
1092         optional => 1,
1093     },
1094     discard => {
1095         type => 'string',
1096         enum => [qw(ignore on)],
1097         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1098         optional => 1,
1099     },
1100     detect_zeroes => {
1101         type => 'boolean',
1102         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1103         optional => 1,
1104     },
1105     serial => {
1106         type => 'string',
1107         format => 'urlencoded',
1108         format_description => 'serial',
1109         maxLength => 20*3, # *3 since it's %xx url enoded
1110         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1111         optional => 1,
1112     },
1113     shared => {
1114         type => 'boolean',
1115         description => 'Mark this locally-managed volume as available on all nodes',
1116         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1117         optional => 1,
1118         default => 0,
1119     }
1120 );
1121
1122 my %iothread_fmt = ( iothread => {
1123         type => 'boolean',
1124         description => "Whether to use iothreads for this drive",
1125         optional => 1,
1126 });
1127
1128 my %model_fmt = (
1129     model => {
1130         type => 'string',
1131         format => 'urlencoded',
1132         format_description => 'model',
1133         maxLength => 40*3, # *3 since it's %xx url enoded
1134         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1135         optional => 1,
1136     },
1137 );
1138
1139 my %queues_fmt = (
1140     queues => {
1141         type => 'integer',
1142         description => "Number of queues.",
1143         minimum => 2,
1144         optional => 1
1145     }
1146 );
1147
1148 my %scsiblock_fmt = (
1149     scsiblock => {
1150         type => 'boolean',
1151         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1152         optional => 1,
1153         default => 0,
1154     },
1155 );
1156
1157 my %ssd_fmt = (
1158     ssd => {
1159         type => 'boolean',
1160         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1161         optional => 1,
1162     },
1163 );
1164
1165 my %wwn_fmt = (
1166     wwn => {
1167         type => 'string',
1168         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1169         format_description => 'wwn',
1170         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1171         optional => 1,
1172     },
1173 );
1174
1175 my $add_throttle_desc = sub {
1176     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1177     my $d = {
1178         type => $type,
1179         format_description => $unit,
1180         description => "Maximum $what in $longunit.",
1181         optional => 1,
1182     };
1183     $d->{minimum} = $minimum if defined($minimum);
1184     $drivedesc_base{$key} = $d;
1185 };
1186 # throughput: (leaky bucket)
1187 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1188 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1189 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1190 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1192 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1193 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1194 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1195 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1196
1197 # pools: (pool of IO before throttling starts taking effect)
1198 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1199 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1200 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1201 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1202 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1203 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1204
1205 # burst lengths
1206 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1210 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1211 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1212
1213 # legacy support
1214 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1215 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1216 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1217 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1218
1219 my $ide_fmt = {
1220     %drivedesc_base,
1221     %model_fmt,
1222     %ssd_fmt,
1223     %wwn_fmt,
1224 };
1225 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1226
1227 my $idedesc = {
1228     optional => 1,
1229     type => 'string', format => $ide_fmt,
1230     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1231 };
1232 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1233
1234 my $scsi_fmt = {
1235     %drivedesc_base,
1236     %iothread_fmt,
1237     %queues_fmt,
1238     %scsiblock_fmt,
1239     %ssd_fmt,
1240     %wwn_fmt,
1241 };
1242 my $scsidesc = {
1243     optional => 1,
1244     type => 'string', format => $scsi_fmt,
1245     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1246 };
1247 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1248
1249 my $sata_fmt = {
1250     %drivedesc_base,
1251     %ssd_fmt,
1252     %wwn_fmt,
1253 };
1254 my $satadesc = {
1255     optional => 1,
1256     type => 'string', format => $sata_fmt,
1257     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1258 };
1259 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1260
1261 my $virtio_fmt = {
1262     %drivedesc_base,
1263     %iothread_fmt,
1264 };
1265 my $virtiodesc = {
1266     optional => 1,
1267     type => 'string', format => $virtio_fmt,
1268     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1269 };
1270 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1271
1272 my $alldrive_fmt = {
1273     %drivedesc_base,
1274     %iothread_fmt,
1275     %model_fmt,
1276     %queues_fmt,
1277     %scsiblock_fmt,
1278     %ssd_fmt,
1279     %wwn_fmt,
1280 };
1281
1282 my $efidisk_fmt = {
1283     volume => { alias => 'file' },
1284     file => {
1285         type => 'string',
1286         format => 'pve-volume-id-or-qm-path',
1287         default_key => 1,
1288         format_description => 'volume',
1289         description => "The drive's backing volume.",
1290     },
1291     format => get_standard_option('pve-qm-image-format'),
1292     size => {
1293         type => 'string',
1294         format => 'disk-size',
1295         format_description => 'DiskSize',
1296         description => "Disk size. This is purely informational and has no effect.",
1297         optional => 1,
1298     },
1299 };
1300
1301 my $efidisk_desc = {
1302     optional => 1,
1303     type => 'string', format => $efidisk_fmt,
1304     description => "Configure a Disk for storing EFI vars",
1305 };
1306
1307 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1308
1309 my $usb_fmt = {
1310     host => {
1311         default_key => 1,
1312         type => 'string', format => 'pve-qm-usb-device',
1313         format_description => 'HOSTUSBDEVICE|spice',
1314         description => <<EODESCR,
1315 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1316
1317  'bus-port(.port)*' (decimal numbers) or
1318  'vendor_id:product_id' (hexadeciaml numbers) or
1319  'spice'
1320
1321 You can use the 'lsusb -t' command to list existing usb devices.
1322
1323 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1324
1325 The value 'spice' can be used to add a usb redirection devices for spice.
1326 EODESCR
1327     },
1328     usb3 => {
1329         optional => 1,
1330         type => 'boolean',
1331         description => "Specifies whether if given host option is a USB3 device or port.",
1332         default => 0,
1333     },
1334 };
1335
1336 my $usbdesc = {
1337     optional => 1,
1338     type => 'string', format => $usb_fmt,
1339     description => "Configure an USB device (n is 0 to 4).",
1340 };
1341 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1342
1343 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1344 my $hostpci_fmt = {
1345     host => {
1346         default_key => 1,
1347         type => 'string',
1348         pattern => qr/$PCIRE(;$PCIRE)*/,
1349         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1350         description => <<EODESCR,
1351 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1352 of PCI virtual functions of the host. HOSTPCIID syntax is:
1353
1354 'bus:dev.func' (hexadecimal numbers)
1355
1356 You can us the 'lspci' command to list existing PCI devices.
1357 EODESCR
1358     },
1359     rombar => {
1360         type => 'boolean',
1361         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1362         optional => 1,
1363         default => 1,
1364     },
1365     romfile => {
1366         type => 'string',
1367         pattern => '[^,;]+',
1368         format_description => 'string',
1369         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1370         optional => 1,
1371     },
1372     pcie => {
1373         type => 'boolean',
1374         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'x-vga' => {
1379         type => 'boolean',
1380         description =>  "Enable vfio-vga device support.",
1381         optional => 1,
1382         default => 0,
1383     },
1384     'mdev' => {
1385         type => 'string',
1386         format_description => 'string',
1387         pattern => '[^/\.:]+',
1388         optional => 1,
1389         description => <<EODESCR
1390 The type of mediated device to use.
1391 An instance of this type will be created on startup of the VM and
1392 will be cleaned up when the VM stops.
1393 EODESCR
1394     }
1395 };
1396 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1397
1398 my $hostpcidesc = {
1399         optional => 1,
1400         type => 'string', format => 'pve-qm-hostpci',
1401         description => "Map host PCI devices into guest.",
1402         verbose_description =>  <<EODESCR,
1403 Map host PCI devices into guest.
1404
1405 NOTE: This option allows direct access to host hardware. So it is no longer
1406 possible to migrate such machines - use with special care.
1407
1408 CAUTION: Experimental! User reported problems with this option.
1409 EODESCR
1410 };
1411 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1412
1413 my $serialdesc = {
1414         optional => 1,
1415         type => 'string',
1416         pattern => '(/dev/.+|socket)',
1417         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1418         verbose_description =>  <<EODESCR,
1419 Create a serial device inside the VM (n is 0 to 3), and pass through a
1420 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1421 host side (use 'qm terminal' to open a terminal connection).
1422
1423 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1424
1425 CAUTION: Experimental! User reported problems with this option.
1426 EODESCR
1427 };
1428
1429 my $paralleldesc= {
1430         optional => 1,
1431         type => 'string',
1432         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1433         description =>  "Map host parallel devices (n is 0 to 2).",
1434         verbose_description =>  <<EODESCR,
1435 Map host parallel devices (n is 0 to 2).
1436
1437 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1438
1439 CAUTION: Experimental! User reported problems with this option.
1440 EODESCR
1441 };
1442
1443 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1444     $confdesc->{"parallel$i"} = $paralleldesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1448     $confdesc->{"serial$i"} = $serialdesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1452     $confdesc->{"hostpci$i"} = $hostpcidesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1456     $drivename_hash->{"ide$i"} = 1;
1457     $confdesc->{"ide$i"} = $idedesc;
1458 }
1459
1460 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1461     $drivename_hash->{"sata$i"} = 1;
1462     $confdesc->{"sata$i"} = $satadesc;
1463 }
1464
1465 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1466     $drivename_hash->{"scsi$i"} = 1;
1467     $confdesc->{"scsi$i"} = $scsidesc ;
1468 }
1469
1470 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1471     $drivename_hash->{"virtio$i"} = 1;
1472     $confdesc->{"virtio$i"} = $virtiodesc;
1473 }
1474
1475 $drivename_hash->{efidisk0} = 1;
1476 $confdesc->{efidisk0} = $efidisk_desc;
1477
1478 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1479     $confdesc->{"usb$i"} = $usbdesc;
1480 }
1481
1482 my $unuseddesc = {
1483     optional => 1,
1484     type => 'string', format => 'pve-volume-id',
1485     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1486 };
1487
1488 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1489     $confdesc->{"unused$i"} = $unuseddesc;
1490 }
1491
1492 my $kvm_api_version = 0;
1493
1494 sub kvm_version {
1495     return $kvm_api_version if $kvm_api_version;
1496
1497     open my $fh, '<', '/dev/kvm'
1498         or return undef;
1499
1500     # 0xae00 => KVM_GET_API_VERSION
1501     $kvm_api_version = ioctl($fh, 0xae00, 0);
1502
1503     return $kvm_api_version;
1504 }
1505
1506 my $kvm_user_version = {};
1507 my $kvm_mtime = {};
1508
1509 sub kvm_user_version {
1510     my ($binary) = @_;
1511
1512     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1513     my $st = stat($binary);
1514
1515     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1516     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1517         $cachedmtime == $st->mtime;
1518
1519     $kvm_user_version->{$binary} = 'unknown';
1520     $kvm_mtime->{$binary} = $st->mtime;
1521
1522     my $code = sub {
1523         my $line = shift;
1524         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1525             $kvm_user_version->{$binary} = $2;
1526         }
1527     };
1528
1529     eval { run_command([$binary, '--version'], outfunc => $code); };
1530     warn $@ if $@;
1531
1532     return $kvm_user_version->{$binary};
1533
1534 }
1535
1536 sub kernel_has_vhost_net {
1537     return -c '/dev/vhost-net';
1538 }
1539
1540 sub valid_drive_names {
1541     # order is important - used to autoselect boot disk
1542     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1543             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1544             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1545             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1546             'efidisk0');
1547 }
1548
1549 sub is_valid_drivename {
1550     my $dev = shift;
1551
1552     return defined($drivename_hash->{$dev});
1553 }
1554
1555 sub option_exists {
1556     my $key = shift;
1557     return defined($confdesc->{$key});
1558 }
1559
1560 my $cdrom_path;
1561 sub get_cdrom_path {
1562
1563     return  $cdrom_path if $cdrom_path;
1564
1565     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1566     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1567     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1568 }
1569
1570 sub get_iso_path {
1571     my ($storecfg, $vmid, $cdrom) = @_;
1572
1573     if ($cdrom eq 'cdrom') {
1574         return get_cdrom_path();
1575     } elsif ($cdrom eq 'none') {
1576         return '';
1577     } elsif ($cdrom =~ m|^/|) {
1578         return $cdrom;
1579     } else {
1580         return PVE::Storage::path($storecfg, $cdrom);
1581     }
1582 }
1583
1584 # try to convert old style file names to volume IDs
1585 sub filename_to_volume_id {
1586     my ($vmid, $file, $media) = @_;
1587
1588      if (!($file eq 'none' || $file eq 'cdrom' ||
1589           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1590
1591         return undef if $file =~ m|/|;
1592
1593         if ($media && $media eq 'cdrom') {
1594             $file = "local:iso/$file";
1595         } else {
1596             $file = "local:$vmid/$file";
1597         }
1598     }
1599
1600     return $file;
1601 }
1602
1603 sub verify_media_type {
1604     my ($opt, $vtype, $media) = @_;
1605
1606     return if !$media;
1607
1608     my $etype;
1609     if ($media eq 'disk') {
1610         $etype = 'images';
1611     } elsif ($media eq 'cdrom') {
1612         $etype = 'iso';
1613     } else {
1614         die "internal error";
1615     }
1616
1617     return if ($vtype eq $etype);
1618
1619     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1620 }
1621
1622 sub cleanup_drive_path {
1623     my ($opt, $storecfg, $drive) = @_;
1624
1625     # try to convert filesystem paths to volume IDs
1626
1627     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1628         ($drive->{file} !~ m|^/dev/.+|) &&
1629         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1630         ($drive->{file} !~ m/^\d+$/)) {
1631         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1632         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1633         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1634         verify_media_type($opt, $vtype, $drive->{media});
1635         $drive->{file} = $volid;
1636     }
1637
1638     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1639 }
1640
1641 sub parse_hotplug_features {
1642     my ($data) = @_;
1643
1644     my $res = {};
1645
1646     return $res if $data eq '0';
1647
1648     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1649
1650     foreach my $feature (PVE::Tools::split_list($data)) {
1651         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1652             $res->{$1} = 1;
1653         } else {
1654             die "invalid hotplug feature '$feature'\n";
1655         }
1656     }
1657     return $res;
1658 }
1659
1660 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1661 sub pve_verify_hotplug_features {
1662     my ($value, $noerr) = @_;
1663
1664     return $value if parse_hotplug_features($value);
1665
1666     return undef if $noerr;
1667
1668     die "unable to parse hotplug option\n";
1669 }
1670
1671 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1672 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1673 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1674 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1675 #        [,iothread=on][,serial=serial][,model=model]
1676
1677 sub parse_drive {
1678     my ($key, $data) = @_;
1679
1680     my ($interface, $index);
1681
1682     if ($key =~ m/^([^\d]+)(\d+)$/) {
1683         $interface = $1;
1684         $index = $2;
1685     } else {
1686         return undef;
1687     }
1688
1689     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1690                                      : $confdesc->{$key}->{format};
1691     if (!$desc) {
1692         warn "invalid drive key: $key\n";
1693         return undef;
1694     }
1695     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1696     return undef if !$res;
1697     $res->{interface} = $interface;
1698     $res->{index} = $index;
1699
1700     my $error = 0;
1701     foreach my $opt (qw(bps bps_rd bps_wr)) {
1702         if (my $bps = defined(delete $res->{$opt})) {
1703             if (defined($res->{"m$opt"})) {
1704                 warn "both $opt and m$opt specified\n";
1705                 ++$error;
1706                 next;
1707             }
1708             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1709         }
1710     }
1711
1712     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1713     for my $requirement (
1714         [mbps_max => 'mbps'],
1715         [mbps_rd_max => 'mbps_rd'],
1716         [mbps_wr_max => 'mbps_wr'],
1717         [miops_max => 'miops'],
1718         [miops_rd_max => 'miops_rd'],
1719         [miops_wr_max => 'miops_wr'],
1720         [bps_max_length => 'mbps_max'],
1721         [bps_rd_max_length => 'mbps_rd_max'],
1722         [bps_wr_max_length => 'mbps_wr_max'],
1723         [iops_max_length => 'iops_max'],
1724         [iops_rd_max_length => 'iops_rd_max'],
1725         [iops_wr_max_length => 'iops_wr_max']) {
1726         my ($option, $requires) = @$requirement;
1727         if ($res->{$option} && !$res->{$requires}) {
1728             warn "$option requires $requires\n";
1729             ++$error;
1730         }
1731     }
1732
1733     return undef if $error;
1734
1735     return undef if $res->{mbps_rd} && $res->{mbps};
1736     return undef if $res->{mbps_wr} && $res->{mbps};
1737     return undef if $res->{iops_rd} && $res->{iops};
1738     return undef if $res->{iops_wr} && $res->{iops};
1739
1740     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1741         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1742         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1743         return undef if $res->{interface} eq 'virtio';
1744     }
1745
1746     if (my $size = $res->{size}) {
1747         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1748     }
1749
1750     return $res;
1751 }
1752
1753 sub print_drive {
1754     my ($vmid, $drive) = @_;
1755     my $data = { %$drive };
1756     delete $data->{$_} for qw(index interface);
1757     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1758 }
1759
1760 sub scsi_inquiry {
1761     my($fh, $noerr) = @_;
1762
1763     my $SG_IO = 0x2285;
1764     my $SG_GET_VERSION_NUM = 0x2282;
1765
1766     my $versionbuf = "\x00" x 8;
1767     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1768     if (!$ret) {
1769         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1770         return undef;
1771     }
1772     my $version = unpack("I", $versionbuf);
1773     if ($version < 30000) {
1774         die "scsi generic interface too old\n"  if !$noerr;
1775         return undef;
1776     }
1777
1778     my $buf = "\x00" x 36;
1779     my $sensebuf = "\x00" x 8;
1780     my $cmd = pack("C x3 C x1", 0x12, 36);
1781
1782     # see /usr/include/scsi/sg.h
1783     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1784
1785     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1786                       length($sensebuf), 0, length($buf), $buf,
1787                       $cmd, $sensebuf, 6000);
1788
1789     $ret = ioctl($fh, $SG_IO, $packet);
1790     if (!$ret) {
1791         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1792         return undef;
1793     }
1794
1795     my @res = unpack($sg_io_hdr_t, $packet);
1796     if ($res[17] || $res[18]) {
1797         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1798         return undef;
1799     }
1800
1801     my $res = {};
1802     (my $byte0, my $byte1, $res->{vendor},
1803      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1804
1805     $res->{removable} = $byte1 & 128 ? 1 : 0;
1806     $res->{type} = $byte0 & 31;
1807
1808     return $res;
1809 }
1810
1811 sub path_is_scsi {
1812     my ($path) = @_;
1813
1814     my $fh = IO::File->new("+<$path") || return undef;
1815     my $res = scsi_inquiry($fh, 1);
1816     close($fh);
1817
1818     return $res;
1819 }
1820
1821 sub print_tabletdevice_full {
1822     my ($conf, $arch) = @_;
1823
1824     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1825
1826     # we use uhci for old VMs because tablet driver was buggy in older qemu
1827     my $usbbus;
1828     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1829         $usbbus = 'ehci';
1830     } else {
1831         $usbbus = 'uhci';
1832     }
1833
1834     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1835 }
1836
1837 sub print_keyboarddevice_full {
1838     my ($conf, $arch, $machine) = @_;
1839
1840     return undef if $arch ne 'aarch64';
1841
1842     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1843 }
1844
1845 sub print_drivedevice_full {
1846     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1847
1848     my $device = '';
1849     my $maxdev = 0;
1850
1851     if ($drive->{interface} eq 'virtio') {
1852         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1853         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1854         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1855     } elsif ($drive->{interface} eq 'scsi') {
1856
1857         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1858         my $unit = $drive->{index} % $maxdev;
1859         my $devicetype = 'hd';
1860         my $path = '';
1861         if (drive_is_cdrom($drive)) {
1862             $devicetype = 'cd';
1863         } else {
1864             if ($drive->{file} =~ m|^/|) {
1865                 $path = $drive->{file};
1866                 if (my $info = path_is_scsi($path)) {
1867                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1868                         $devicetype = 'block';
1869                     } elsif ($info->{type} == 1) { # tape
1870                         $devicetype = 'generic';
1871                     }
1872                 }
1873             } else {
1874                  $path = PVE::Storage::path($storecfg, $drive->{file});
1875             }
1876
1877             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1878             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1879             if ($path =~ m/^iscsi\:\/\// &&
1880                !min_version($version, 4, 1)) {
1881                 $devicetype = 'generic';
1882             }
1883         }
1884
1885         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1886             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1887         } else {
1888             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1889         }
1890
1891         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1892             $device .= ",rotation_rate=1";
1893         }
1894         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1895
1896     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1897         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1898         my $controller = int($drive->{index} / $maxdev);
1899         my $unit = $drive->{index} % $maxdev;
1900         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1901
1902         $device = "ide-$devicetype";
1903         if ($drive->{interface} eq 'ide') {
1904             $device .= ",bus=ide.$controller,unit=$unit";
1905         } else {
1906             $device .= ",bus=ahci$controller.$unit";
1907         }
1908         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1909
1910         if ($devicetype eq 'hd') {
1911             if (my $model = $drive->{model}) {
1912                 $model = URI::Escape::uri_unescape($model);
1913                 $device .= ",model=$model";
1914             }
1915             if ($drive->{ssd}) {
1916                 $device .= ",rotation_rate=1";
1917             }
1918         }
1919         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1920     } elsif ($drive->{interface} eq 'usb') {
1921         die "implement me";
1922         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1923     } else {
1924         die "unsupported interface type";
1925     }
1926
1927     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1928
1929     if (my $serial = $drive->{serial}) {
1930         $serial = URI::Escape::uri_unescape($serial);
1931         $device .= ",serial=$serial";
1932     }
1933
1934
1935     return $device;
1936 }
1937
1938 sub get_initiator_name {
1939     my $initiator;
1940
1941     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1942     while (defined(my $line = <$fh>)) {
1943         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1944         $initiator = $1;
1945         last;
1946     }
1947     $fh->close();
1948
1949     return $initiator;
1950 }
1951
1952 sub print_drive_full {
1953     my ($storecfg, $vmid, $drive) = @_;
1954
1955     my $path;
1956     my $volid = $drive->{file};
1957     my $format;
1958
1959     if (drive_is_cdrom($drive)) {
1960         $path = get_iso_path($storecfg, $vmid, $volid);
1961     } else {
1962         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1963         if ($storeid) {
1964             $path = PVE::Storage::path($storecfg, $volid);
1965             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1966             $format = qemu_img_format($scfg, $volname);
1967         } else {
1968             $path = $volid;
1969             $format = "raw";
1970         }
1971    }
1972
1973     my $opts = '';
1974     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1975     foreach my $o (@qemu_drive_options) {
1976         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1977     }
1978
1979     # snapshot only accepts on|off
1980     if (defined($drive->{snapshot})) {
1981         my $v = $drive->{snapshot} ? 'on' : 'off';
1982         $opts .= ",snapshot=$v";
1983     }
1984
1985     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1986         my ($dir, $qmpname) = @$type;
1987         if (my $v = $drive->{"mbps$dir"}) {
1988             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1989         }
1990         if (my $v = $drive->{"mbps${dir}_max"}) {
1991             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1992         }
1993         if (my $v = $drive->{"bps${dir}_max_length"}) {
1994             $opts .= ",throttling.bps$qmpname-max-length=$v";
1995         }
1996         if (my $v = $drive->{"iops${dir}"}) {
1997             $opts .= ",throttling.iops$qmpname=$v";
1998         }
1999         if (my $v = $drive->{"iops${dir}_max"}) {
2000             $opts .= ",throttling.iops$qmpname-max=$v";
2001         }
2002         if (my $v = $drive->{"iops${dir}_max_length"}) {
2003             $opts .= ",throttling.iops$qmpname-max-length=$v";
2004         }
2005     }
2006
2007     $opts .= ",format=$format" if $format && !$drive->{format};
2008
2009     my $cache_direct = 0;
2010
2011     if (my $cache = $drive->{cache}) {
2012         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2013     } elsif (!drive_is_cdrom($drive)) {
2014         $opts .= ",cache=none";
2015         $cache_direct = 1;
2016     }
2017
2018     # aio native works only with O_DIRECT
2019     if (!$drive->{aio}) {
2020         if($cache_direct) {
2021             $opts .= ",aio=native";
2022         } else {
2023             $opts .= ",aio=threads";
2024         }
2025     }
2026
2027     if (!drive_is_cdrom($drive)) {
2028         my $detectzeroes;
2029         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2030             $detectzeroes = 'off';
2031         } elsif ($drive->{discard}) {
2032             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2033         } else {
2034             # This used to be our default with discard not being specified:
2035             $detectzeroes = 'on';
2036         }
2037         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2038     }
2039
2040     my $pathinfo = $path ? "file=$path," : '';
2041
2042     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2043 }
2044
2045 sub print_netdevice_full {
2046     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2047
2048     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2049
2050     my $device = $net->{model};
2051     if ($net->{model} eq 'virtio') {
2052          $device = 'virtio-net-pci';
2053      };
2054
2055     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2056     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2057     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2058         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2059         my $vectors = $net->{queues} * 2 + 2;
2060         $tmpstr .= ",vectors=$vectors,mq=on";
2061     }
2062     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2063
2064     if ($use_old_bios_files) {
2065         my $romfile;
2066         if ($device eq 'virtio-net-pci') {
2067             $romfile = 'pxe-virtio.rom';
2068         } elsif ($device eq 'e1000') {
2069             $romfile = 'pxe-e1000.rom';
2070         } elsif ($device eq 'ne2k') {
2071             $romfile = 'pxe-ne2k_pci.rom';
2072         } elsif ($device eq 'pcnet') {
2073             $romfile = 'pxe-pcnet.rom';
2074         } elsif ($device eq 'rtl8139') {
2075             $romfile = 'pxe-rtl8139.rom';
2076         }
2077         $tmpstr .= ",romfile=$romfile" if $romfile;
2078     }
2079
2080     return $tmpstr;
2081 }
2082
2083 sub print_netdev_full {
2084     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2085
2086     my $i = '';
2087     if ($netid =~ m/^net(\d+)$/) {
2088         $i = int($1);
2089     }
2090
2091     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2092
2093     my $ifname = "tap${vmid}i$i";
2094
2095     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2096     die "interface name '$ifname' is too long (max 15 character)\n"
2097         if length($ifname) >= 16;
2098
2099     my $vhostparam = '';
2100     if (is_native($arch)) {
2101         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2102     }
2103
2104     my $vmname = $conf->{name} || "vm$vmid";
2105
2106     my $netdev = "";
2107     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2108
2109     if ($net->{bridge}) {
2110         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2111     } else {
2112         $netdev = "type=user,id=$netid,hostname=$vmname";
2113     }
2114
2115     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2116
2117     return $netdev;
2118 }
2119
2120
2121 sub print_cpu_device {
2122     my ($conf, $id) = @_;
2123
2124     my $kvm = $conf->{kvm} // 1;
2125     my $cpu = $kvm ? "kvm64" : "qemu64";
2126     if (my $cputype = $conf->{cpu}) {
2127         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2128             or die "Cannot parse cpu description: $cputype\n";
2129         $cpu = $cpuconf->{cputype};
2130     }
2131
2132     my $cores = $conf->{cores} || 1;
2133
2134     my $current_core = ($id - 1) % $cores;
2135     my $current_socket = int(($id - 1 - $current_core)/$cores);
2136
2137     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2138 }
2139
2140 my $vga_map = {
2141     'cirrus' => 'cirrus-vga',
2142     'std' => 'VGA',
2143     'vmware' => 'vmware-svga',
2144     'virtio' => 'virtio-vga',
2145 };
2146
2147 sub print_vga_device {
2148     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2149
2150     my $type = $vga_map->{$vga->{type}};
2151     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2152         $type = 'virtio-gpu';
2153     }
2154     my $vgamem_mb = $vga->{memory};
2155
2156     my $max_outputs = '';
2157     if ($qxlnum) {
2158         $type = $id ? 'qxl' : 'qxl-vga';
2159
2160         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2161             # set max outputs so linux can have up to 4 qxl displays with one device
2162             if (min_version($machine_version, 4, 1)) {
2163                 $max_outputs = ",max_outputs=4";
2164             }
2165         }
2166     }
2167
2168     die "no devicetype for $vga->{type}\n" if !$type;
2169
2170     my $memory = "";
2171     if ($vgamem_mb) {
2172         if ($vga->{type} eq 'virtio') {
2173             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2174             $memory = ",max_hostmem=$bytes";
2175         } elsif ($qxlnum) {
2176             # from https://www.spice-space.org/multiple-monitors.html
2177             $memory = ",vgamem_mb=$vga->{memory}";
2178             my $ram = $vgamem_mb * 4;
2179             my $vram = $vgamem_mb * 2;
2180             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2181         } else {
2182             $memory = ",vgamem_mb=$vga->{memory}";
2183         }
2184     } elsif ($qxlnum && $id) {
2185         $memory = ",ram_size=67108864,vram_size=33554432";
2186     }
2187
2188     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2189     my $vgaid = "vga" . ($id // '');
2190     my $pciaddr;
2191
2192     if ($q35 && $vgaid eq 'vga') {
2193         # the first display uses pcie.0 bus on q35 machines
2194         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2195     } else {
2196         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2197     }
2198
2199     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2200 }
2201
2202 sub drive_is_cloudinit {
2203     my ($drive) = @_;
2204     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2205 }
2206
2207 sub drive_is_cdrom {
2208     my ($drive, $exclude_cloudinit) = @_;
2209
2210     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2211
2212     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2213
2214 }
2215
2216 sub parse_number_sets {
2217     my ($set) = @_;
2218     my $res = [];
2219     foreach my $part (split(/;/, $set)) {
2220         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2221             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2222             push @$res, [ $1, $2 ];
2223         } else {
2224             die "invalid range: $part\n";
2225         }
2226     }
2227     return $res;
2228 }
2229
2230 sub parse_numa {
2231     my ($data) = @_;
2232
2233     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2234     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2235     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2236     return $res;
2237 }
2238
2239 sub parse_hostpci {
2240     my ($value) = @_;
2241
2242     return undef if !$value;
2243
2244     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2245
2246     my @idlist = split(/;/, $res->{host});
2247     delete $res->{host};
2248     foreach my $id (@idlist) {
2249         if ($id =~ m/\./) { # full id 00:00.1
2250             push @{$res->{pciid}}, {
2251                 id => $id,
2252             };
2253         } else { # partial id 00:00
2254             $res->{pciid} = PVE::SysFSTools::lspci($id);
2255         }
2256     }
2257     return $res;
2258 }
2259
2260 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2261 sub parse_net {
2262     my ($data) = @_;
2263
2264     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2265     if ($@) {
2266         warn $@;
2267         return undef;
2268     }
2269     if (!defined($res->{macaddr})) {
2270         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2272     }
2273     return $res;
2274 }
2275
2276 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2277 sub parse_ipconfig {
2278     my ($data) = @_;
2279
2280     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2281     if ($@) {
2282         warn $@;
2283         return undef;
2284     }
2285
2286     if ($res->{gw} && !$res->{ip}) {
2287         warn 'gateway specified without specifying an IP address';
2288         return undef;
2289     }
2290     if ($res->{gw6} && !$res->{ip6}) {
2291         warn 'IPv6 gateway specified without specifying an IPv6 address';
2292         return undef;
2293     }
2294     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2295         warn 'gateway specified together with DHCP';
2296         return undef;
2297     }
2298     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2299         # gw6 + auto/dhcp
2300         warn "IPv6 gateway specified together with $res->{ip6} address";
2301         return undef;
2302     }
2303
2304     if (!$res->{ip} && !$res->{ip6}) {
2305         return { ip => 'dhcp', ip6 => 'dhcp' };
2306     }
2307
2308     return $res;
2309 }
2310
2311 sub print_net {
2312     my $net = shift;
2313
2314     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2315 }
2316
2317 sub add_random_macs {
2318     my ($settings) = @_;
2319
2320     foreach my $opt (keys %$settings) {
2321         next if $opt !~ m/^net(\d+)$/;
2322         my $net = parse_net($settings->{$opt});
2323         next if !$net;
2324         $settings->{$opt} = print_net($net);
2325     }
2326 }
2327
2328 sub vm_is_volid_owner {
2329     my ($storecfg, $vmid, $volid) = @_;
2330
2331     if ($volid !~  m|^/|) {
2332         my ($path, $owner);
2333         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2334         if ($owner && ($owner == $vmid)) {
2335             return 1;
2336         }
2337     }
2338
2339     return undef;
2340 }
2341
2342 sub vmconfig_register_unused_drive {
2343     my ($storecfg, $vmid, $conf, $drive) = @_;
2344
2345     if (drive_is_cloudinit($drive)) {
2346         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2347         warn $@ if $@;
2348     } elsif (!drive_is_cdrom($drive)) {
2349         my $volid = $drive->{file};
2350         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2351             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2352         }
2353     }
2354 }
2355
2356 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2357 my $smbios1_fmt = {
2358     uuid => {
2359         type => 'string',
2360         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2361         format_description => 'UUID',
2362         description => "Set SMBIOS1 UUID.",
2363         optional => 1,
2364     },
2365     version => {
2366         type => 'string',
2367         pattern => '[A-Za-z0-9+\/]+={0,2}',
2368         format_description => 'Base64 encoded string',
2369         description => "Set SMBIOS1 version.",
2370         optional => 1,
2371     },
2372     serial => {
2373         type => 'string',
2374         pattern => '[A-Za-z0-9+\/]+={0,2}',
2375         format_description => 'Base64 encoded string',
2376         description => "Set SMBIOS1 serial number.",
2377         optional => 1,
2378     },
2379     manufacturer => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 manufacturer.",
2384         optional => 1,
2385     },
2386     product => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 product ID.",
2391         optional => 1,
2392     },
2393     sku => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 SKU string.",
2398         optional => 1,
2399     },
2400     family => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 family string.",
2405         optional => 1,
2406     },
2407     base64 => {
2408         type => 'boolean',
2409         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2410         optional => 1,
2411     },
2412 };
2413
2414 sub parse_smbios1 {
2415     my ($data) = @_;
2416
2417     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2418     warn $@ if $@;
2419     return $res;
2420 }
2421
2422 sub print_smbios1 {
2423     my ($smbios1) = @_;
2424     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2425 }
2426
2427 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2428
2429 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2430 sub verify_bootdisk {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if is_valid_drivename($value);
2434
2435     return undef if $noerr;
2436
2437     die "invalid boot disk '$value'\n";
2438 }
2439
2440 sub parse_watchdog {
2441     my ($value) = @_;
2442
2443     return undef if !$value;
2444
2445     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 sub parse_guest_agent {
2451     my ($value) = @_;
2452
2453     return {} if !defined($value->{agent});
2454
2455     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2456     warn $@ if $@;
2457
2458     # if the agent is disabled ignore the other potentially set properties
2459     return {} if !$res->{enabled};
2460     return $res;
2461 }
2462
2463 sub parse_vga {
2464     my ($value) = @_;
2465
2466     return {} if !$value;
2467     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2473 sub verify_usb_device {
2474     my ($value, $noerr) = @_;
2475
2476     return $value if parse_usb_device($value);
2477
2478     return undef if $noerr;
2479
2480     die "unable to parse usb device\n";
2481 }
2482
2483 # add JSON properties for create and set function
2484 sub json_config_properties {
2485     my $prop = shift;
2486
2487     foreach my $opt (keys %$confdesc) {
2488         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489         $prop->{$opt} = $confdesc->{$opt};
2490     }
2491
2492     return $prop;
2493 }
2494
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties {
2497
2498     return dclone($confdesc_cloudinit);
2499 }
2500
2501 sub check_type {
2502     my ($key, $value) = @_;
2503
2504     die "unknown setting '$key'\n" if !$confdesc->{$key};
2505
2506     my $type = $confdesc->{$key}->{type};
2507
2508     if (!defined($value)) {
2509         die "got undefined value\n";
2510     }
2511
2512     if ($value =~ m/[\n\r]/) {
2513         die "property contains a line feed\n";
2514     }
2515
2516     if ($type eq 'boolean') {
2517         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519         die "type check ('boolean') failed - got '$value'\n";
2520     } elsif ($type eq 'integer') {
2521         return int($1) if $value =~ m/^(\d+)$/;
2522         die "type check ('integer') failed - got '$value'\n";
2523     } elsif ($type eq 'number') {
2524         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525         die "type check ('number') failed - got '$value'\n";
2526     } elsif ($type eq 'string') {
2527         if (my $fmt = $confdesc->{$key}->{format}) {
2528             PVE::JSONSchema::check_format($fmt, $value);
2529             return $value;
2530         }
2531         $value =~ s/^\"(.*)\"$/$1/;
2532         return $value;
2533     } else {
2534         die "internal error"
2535     }
2536 }
2537
2538 sub destroy_vm {
2539     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2540
2541     my $conf = PVE::QemuConfig->load_config($vmid);
2542
2543     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2544
2545     if ($conf->{template}) {
2546         # check if any base image is still used by a linked clone
2547         foreach_drive($conf, sub {
2548                 my ($ds, $drive) = @_;
2549                 return if drive_is_cdrom($drive);
2550
2551                 my $volid = $drive->{file};
2552                 return if !$volid || $volid =~ m|^/|;
2553
2554                 die "base volume '$volid' is still in use by linked cloned\n"
2555                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2556
2557         });
2558     }
2559
2560     # only remove disks owned by this VM
2561     foreach_drive($conf, sub {
2562         my ($ds, $drive) = @_;
2563         return if drive_is_cdrom($drive, 1);
2564
2565         my $volid = $drive->{file};
2566         return if !$volid || $volid =~ m|^/|;
2567
2568         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2569         return if !$path || !$owner || ($owner != $vmid);
2570
2571         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2572         warn "Could not remove disk '$volid', check manually: $@" if $@;
2573     });
2574
2575     # also remove unused disk
2576     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2577     PVE::Storage::foreach_volid($vmdisks, sub {
2578         my ($volid, $sid, $volname, $d) = @_;
2579         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2580         warn $@ if $@;
2581     });
2582
2583     if (defined $replacement_conf) {
2584         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2585     } else {
2586         PVE::QemuConfig->destroy_config($vmid);
2587     }
2588 }
2589
2590 sub parse_vm_config {
2591     my ($filename, $raw) = @_;
2592
2593     return undef if !defined($raw);
2594
2595     my $res = {
2596         digest => Digest::SHA::sha1_hex($raw),
2597         snapshots => {},
2598         pending => {},
2599     };
2600
2601     $filename =~ m|/qemu-server/(\d+)\.conf$|
2602         || die "got strange filename '$filename'";
2603
2604     my $vmid = $1;
2605
2606     my $conf = $res;
2607     my $descr;
2608     my $section = '';
2609
2610     my @lines = split(/\n/, $raw);
2611     foreach my $line (@lines) {
2612         next if $line =~ m/^\s*$/;
2613
2614         if ($line =~ m/^\[PENDING\]\s*$/i) {
2615             $section = 'pending';
2616             if (defined($descr)) {
2617                 $descr =~ s/\s+$//;
2618                 $conf->{description} = $descr;
2619             }
2620             $descr = undef;
2621             $conf = $res->{$section} = {};
2622             next;
2623
2624         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2625             $section = $1;
2626             if (defined($descr)) {
2627                 $descr =~ s/\s+$//;
2628                 $conf->{description} = $descr;
2629             }
2630             $descr = undef;
2631             $conf = $res->{snapshots}->{$section} = {};
2632             next;
2633         }
2634
2635         if ($line =~ m/^\#(.*)\s*$/) {
2636             $descr = '' if !defined($descr);
2637             $descr .= PVE::Tools::decode_text($1) . "\n";
2638             next;
2639         }
2640
2641         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2642             $descr = '' if !defined($descr);
2643             $descr .= PVE::Tools::decode_text($2);
2644         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2645             $conf->{snapstate} = $1;
2646         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2647             my $key = $1;
2648             my $value = $2;
2649             $conf->{$key} = $value;
2650         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2651             my $value = $1;
2652             if ($section eq 'pending') {
2653                 $conf->{delete} = $value; # we parse this later
2654             } else {
2655                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2656             }
2657         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2658             my $key = $1;
2659             my $value = $2;
2660             eval { $value = check_type($key, $value); };
2661             if ($@) {
2662                 warn "vm $vmid - unable to parse value of '$key' - $@";
2663             } else {
2664                 $key = 'ide2' if $key eq 'cdrom';
2665                 my $fmt = $confdesc->{$key}->{format};
2666                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2667                     my $v = parse_drive($key, $value);
2668                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2669                         $v->{file} = $volid;
2670                         $value = print_drive($vmid, $v);
2671                     } else {
2672                         warn "vm $vmid - unable to parse value of '$key'\n";
2673                         next;
2674                     }
2675                 }
2676
2677                 $conf->{$key} = $value;
2678             }
2679         }
2680     }
2681
2682     if (defined($descr)) {
2683         $descr =~ s/\s+$//;
2684         $conf->{description} = $descr;
2685     }
2686     delete $res->{snapstate}; # just to be sure
2687
2688     return $res;
2689 }
2690
2691 sub write_vm_config {
2692     my ($filename, $conf) = @_;
2693
2694     delete $conf->{snapstate}; # just to be sure
2695
2696     if ($conf->{cdrom}) {
2697         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2698         $conf->{ide2} = $conf->{cdrom};
2699         delete $conf->{cdrom};
2700     }
2701
2702     # we do not use 'smp' any longer
2703     if ($conf->{sockets}) {
2704         delete $conf->{smp};
2705     } elsif ($conf->{smp}) {
2706         $conf->{sockets} = $conf->{smp};
2707         delete $conf->{cores};
2708         delete $conf->{smp};
2709     }
2710
2711     my $used_volids = {};
2712
2713     my $cleanup_config = sub {
2714         my ($cref, $pending, $snapname) = @_;
2715
2716         foreach my $key (keys %$cref) {
2717             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2718                 $key eq 'snapstate' || $key eq 'pending';
2719             my $value = $cref->{$key};
2720             if ($key eq 'delete') {
2721                 die "propertry 'delete' is only allowed in [PENDING]\n"
2722                     if !$pending;
2723                 # fixme: check syntax?
2724                 next;
2725             }
2726             eval { $value = check_type($key, $value); };
2727             die "unable to parse value of '$key' - $@" if $@;
2728
2729             $cref->{$key} = $value;
2730
2731             if (!$snapname && is_valid_drivename($key)) {
2732                 my $drive = parse_drive($key, $value);
2733                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2734             }
2735         }
2736     };
2737
2738     &$cleanup_config($conf);
2739
2740     &$cleanup_config($conf->{pending}, 1);
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2744         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2745     }
2746
2747     # remove 'unusedX' settings if we re-add a volume
2748     foreach my $key (keys %$conf) {
2749         my $value = $conf->{$key};
2750         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2751             delete $conf->{$key};
2752         }
2753     }
2754
2755     my $generate_raw_config = sub {
2756         my ($conf, $pending) = @_;
2757
2758         my $raw = '';
2759
2760         # add description as comment to top of file
2761         if (defined(my $descr = $conf->{description})) {
2762             if ($descr) {
2763                 foreach my $cl (split(/\n/, $descr)) {
2764                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2765                 }
2766             } else {
2767                 $raw .= "#\n" if $pending;
2768             }
2769         }
2770
2771         foreach my $key (sort keys %$conf) {
2772             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2773             $raw .= "$key: $conf->{$key}\n";
2774         }
2775         return $raw;
2776     };
2777
2778     my $raw = &$generate_raw_config($conf);
2779
2780     if (scalar(keys %{$conf->{pending}})){
2781         $raw .= "\n[PENDING]\n";
2782         $raw .= &$generate_raw_config($conf->{pending}, 1);
2783     }
2784
2785     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2786         $raw .= "\n[$snapname]\n";
2787         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2788     }
2789
2790     return $raw;
2791 }
2792
2793 sub load_defaults {
2794
2795     my $res = {};
2796
2797     # we use static defaults from our JSON schema configuration
2798     foreach my $key (keys %$confdesc) {
2799         if (defined(my $default = $confdesc->{$key}->{default})) {
2800             $res->{$key} = $default;
2801         }
2802     }
2803
2804     return $res;
2805 }
2806
2807 sub config_list {
2808     my $vmlist = PVE::Cluster::get_vmlist();
2809     my $res = {};
2810     return $res if !$vmlist || !$vmlist->{ids};
2811     my $ids = $vmlist->{ids};
2812
2813     foreach my $vmid (keys %$ids) {
2814         my $d = $ids->{$vmid};
2815         next if !$d->{node} || $d->{node} ne $nodename;
2816         next if !$d->{type} || $d->{type} ne 'qemu';
2817         $res->{$vmid}->{exists} = 1;
2818     }
2819     return $res;
2820 }
2821
2822 # test if VM uses local resources (to prevent migration)
2823 sub check_local_resources {
2824     my ($conf, $noerr) = @_;
2825
2826     my @loc_res = ();
2827
2828     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2829     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2830
2831     push @loc_res, "ivshmem" if $conf->{ivshmem};
2832
2833     foreach my $k (keys %$conf) {
2834         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2835         # sockets are safe: they will recreated be on the target side post-migrate
2836         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2837         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2838     }
2839
2840     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2841
2842     return \@loc_res;
2843 }
2844
2845 # check if used storages are available on all nodes (use by migrate)
2846 sub check_storage_availability {
2847     my ($storecfg, $conf, $node) = @_;
2848
2849     foreach_drive($conf, sub {
2850         my ($ds, $drive) = @_;
2851
2852         my $volid = $drive->{file};
2853         return if !$volid;
2854
2855         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2856         return if !$sid;
2857
2858         # check if storage is available on both nodes
2859         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2860         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2861    });
2862 }
2863
2864 # list nodes where all VM images are available (used by has_feature API)
2865 sub shared_nodes {
2866     my ($conf, $storecfg) = @_;
2867
2868     my $nodelist = PVE::Cluster::get_nodelist();
2869     my $nodehash = { map { $_ => 1 } @$nodelist };
2870     my $nodename = PVE::INotify::nodename();
2871
2872     foreach_drive($conf, sub {
2873         my ($ds, $drive) = @_;
2874
2875         my $volid = $drive->{file};
2876         return if !$volid;
2877
2878         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2879         if ($storeid) {
2880             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2881             if ($scfg->{disable}) {
2882                 $nodehash = {};
2883             } elsif (my $avail = $scfg->{nodes}) {
2884                 foreach my $node (keys %$nodehash) {
2885                     delete $nodehash->{$node} if !$avail->{$node};
2886                 }
2887             } elsif (!$scfg->{shared}) {
2888                 foreach my $node (keys %$nodehash) {
2889                     delete $nodehash->{$node} if $node ne $nodename
2890                 }
2891             }
2892         }
2893     });
2894
2895     return $nodehash
2896 }
2897
2898 sub check_local_storage_availability {
2899     my ($conf, $storecfg) = @_;
2900
2901     my $nodelist = PVE::Cluster::get_nodelist();
2902     my $nodehash = { map { $_ => {} } @$nodelist };
2903
2904     foreach_drive($conf, sub {
2905         my ($ds, $drive) = @_;
2906
2907         my $volid = $drive->{file};
2908         return if !$volid;
2909
2910         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2911         if ($storeid) {
2912             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2913
2914             if ($scfg->{disable}) {
2915                 foreach my $node (keys %$nodehash) {
2916                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2917                 }
2918             } elsif (my $avail = $scfg->{nodes}) {
2919                 foreach my $node (keys %$nodehash) {
2920                     if (!$avail->{$node}) {
2921                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2922                     }
2923                 }
2924             }
2925         }
2926     });
2927
2928     foreach my $node (values %$nodehash) {
2929         if (my $unavail = $node->{unavailable_storages}) {
2930             $node->{unavailable_storages} = [ sort keys %$unavail ];
2931         }
2932     }
2933
2934     return $nodehash
2935 }
2936
2937 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2938 sub check_running {
2939     my ($vmid, $nocheck, $node) = @_;
2940
2941     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2942     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2943 }
2944
2945 sub vzlist {
2946
2947     my $vzlist = config_list();
2948
2949     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2950
2951     while (defined(my $de = $fd->read)) {
2952         next if $de !~ m/^(\d+)\.pid$/;
2953         my $vmid = $1;
2954         next if !defined($vzlist->{$vmid});
2955         if (my $pid = check_running($vmid)) {
2956             $vzlist->{$vmid}->{pid} = $pid;
2957         }
2958     }
2959
2960     return $vzlist;
2961 }
2962
2963 sub disksize {
2964     my ($storecfg, $conf) = @_;
2965
2966     my $bootdisk = $conf->{bootdisk};
2967     return undef if !$bootdisk;
2968     return undef if !is_valid_drivename($bootdisk);
2969
2970     return undef if !$conf->{$bootdisk};
2971
2972     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2973     return undef if !defined($drive);
2974
2975     return undef if drive_is_cdrom($drive);
2976
2977     my $volid = $drive->{file};
2978     return undef if !$volid;
2979
2980     return $drive->{size};
2981 }
2982
2983 our $vmstatus_return_properties = {
2984     vmid => get_standard_option('pve-vmid'),
2985     status => {
2986         description => "Qemu process status.",
2987         type => 'string',
2988         enum => ['stopped', 'running'],
2989     },
2990     maxmem => {
2991         description => "Maximum memory in bytes.",
2992         type => 'integer',
2993         optional => 1,
2994         renderer => 'bytes',
2995     },
2996     maxdisk => {
2997         description => "Root disk size in bytes.",
2998         type => 'integer',
2999         optional => 1,
3000         renderer => 'bytes',
3001     },
3002     name => {
3003         description => "VM name.",
3004         type => 'string',
3005         optional => 1,
3006     },
3007     qmpstatus => {
3008         description => "Qemu QMP agent status.",
3009         type => 'string',
3010         optional => 1,
3011     },
3012     pid => {
3013         description => "PID of running qemu process.",
3014         type => 'integer',
3015         optional => 1,
3016     },
3017     uptime => {
3018         description => "Uptime.",
3019         type => 'integer',
3020         optional => 1,
3021         renderer => 'duration',
3022     },
3023     cpus => {
3024         description => "Maximum usable CPUs.",
3025         type => 'number',
3026         optional => 1,
3027     },
3028     lock => {
3029         description => "The current config lock, if any.",
3030         type => 'string',
3031         optional => 1,
3032     },
3033     tags => {
3034         description  => "The current configured tags, if any",
3035         type => 'string',
3036         optional => 1,
3037     },
3038 };
3039
3040 my $last_proc_pid_stat;
3041
3042 # get VM status information
3043 # This must be fast and should not block ($full == false)
3044 # We only query KVM using QMP if $full == true (this can be slow)
3045 sub vmstatus {
3046     my ($opt_vmid, $full) = @_;
3047
3048     my $res = {};
3049
3050     my $storecfg = PVE::Storage::config();
3051
3052     my $list = vzlist();
3053     my $defaults = load_defaults();
3054
3055     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3056
3057     my $cpucount = $cpuinfo->{cpus} || 1;
3058
3059     foreach my $vmid (keys %$list) {
3060         next if $opt_vmid && ($vmid ne $opt_vmid);
3061
3062         my $conf = PVE::QemuConfig->load_config($vmid);
3063
3064         my $d = { vmid => $vmid };
3065         $d->{pid} = $list->{$vmid}->{pid};
3066
3067         # fixme: better status?
3068         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3069
3070         my $size = disksize($storecfg, $conf);
3071         if (defined($size)) {
3072             $d->{disk} = 0; # no info available
3073             $d->{maxdisk} = $size;
3074         } else {
3075             $d->{disk} = 0;
3076             $d->{maxdisk} = 0;
3077         }
3078
3079         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3080             * ($conf->{cores} || $defaults->{cores});
3081         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3082         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3083
3084         $d->{name} = $conf->{name} || "VM $vmid";
3085         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3086             : $defaults->{memory}*(1024*1024);
3087
3088         if ($conf->{balloon}) {
3089             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3090             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3091                 : $defaults->{shares};
3092         }
3093
3094         $d->{uptime} = 0;
3095         $d->{cpu} = 0;
3096         $d->{mem} = 0;
3097
3098         $d->{netout} = 0;
3099         $d->{netin} = 0;
3100
3101         $d->{diskread} = 0;
3102         $d->{diskwrite} = 0;
3103
3104         $d->{template} = PVE::QemuConfig->is_template($conf);
3105
3106         $d->{serial} = 1 if conf_has_serial($conf);
3107         $d->{lock} = $conf->{lock} if $conf->{lock};
3108         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3109
3110         $res->{$vmid} = $d;
3111     }
3112
3113     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3114     foreach my $dev (keys %$netdev) {
3115         next if $dev !~ m/^tap([1-9]\d*)i/;
3116         my $vmid = $1;
3117         my $d = $res->{$vmid};
3118         next if !$d;
3119
3120         $d->{netout} += $netdev->{$dev}->{receive};
3121         $d->{netin} += $netdev->{$dev}->{transmit};
3122
3123         if ($full) {
3124             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3125             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3126         }
3127
3128     }
3129
3130     my $ctime = gettimeofday;
3131
3132     foreach my $vmid (keys %$list) {
3133
3134         my $d = $res->{$vmid};
3135         my $pid = $d->{pid};
3136         next if !$pid;
3137
3138         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3139         next if !$pstat; # not running
3140
3141         my $used = $pstat->{utime} + $pstat->{stime};
3142
3143         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3144
3145         if ($pstat->{vsize}) {
3146             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3147         }
3148
3149         my $old = $last_proc_pid_stat->{$pid};
3150         if (!$old) {
3151             $last_proc_pid_stat->{$pid} = {
3152                 time => $ctime,
3153                 used => $used,
3154                 cpu => 0,
3155             };
3156             next;
3157         }
3158
3159         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3160
3161         if ($dtime > 1000) {
3162             my $dutime = $used -  $old->{used};
3163
3164             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3165             $last_proc_pid_stat->{$pid} = {
3166                 time => $ctime,
3167                 used => $used,
3168                 cpu => $d->{cpu},
3169             };
3170         } else {
3171             $d->{cpu} = $old->{cpu};
3172         }
3173     }
3174
3175     return $res if !$full;
3176
3177     my $qmpclient = PVE::QMPClient->new();
3178
3179     my $ballooncb = sub {
3180         my ($vmid, $resp) = @_;
3181
3182         my $info = $resp->{'return'};
3183         return if !$info->{max_mem};
3184
3185         my $d = $res->{$vmid};
3186
3187         # use memory assigned to VM
3188         $d->{maxmem} = $info->{max_mem};
3189         $d->{balloon} = $info->{actual};
3190
3191         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3192             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3193             $d->{freemem} = $info->{free_mem};
3194         }
3195
3196         $d->{ballooninfo} = $info;
3197     };
3198
3199     my $blockstatscb = sub {
3200         my ($vmid, $resp) = @_;
3201         my $data = $resp->{'return'} || [];
3202         my $totalrdbytes = 0;
3203         my $totalwrbytes = 0;
3204
3205         for my $blockstat (@$data) {
3206             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3207             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3208
3209             $blockstat->{device} =~ s/drive-//;
3210             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3211         }
3212         $res->{$vmid}->{diskread} = $totalrdbytes;
3213         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3214     };
3215
3216     my $statuscb = sub {
3217         my ($vmid, $resp) = @_;
3218
3219         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3220         # this fails if ballon driver is not loaded, so this must be
3221         # the last commnand (following command are aborted if this fails).
3222         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3223
3224         my $status = 'unknown';
3225         if (!defined($status = $resp->{'return'}->{status})) {
3226             warn "unable to get VM status\n";
3227             return;
3228         }
3229
3230         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3231     };
3232
3233     foreach my $vmid (keys %$list) {
3234         next if $opt_vmid && ($vmid ne $opt_vmid);
3235         next if !$res->{$vmid}->{pid}; # not running
3236         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3237     }
3238
3239     $qmpclient->queue_execute(undef, 2);
3240
3241     foreach my $vmid (keys %$list) {
3242         next if $opt_vmid && ($vmid ne $opt_vmid);
3243         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3244     }
3245
3246     return $res;
3247 }
3248
3249 sub foreach_drive {
3250     my ($conf, $func, @param) = @_;
3251
3252     foreach my $ds (valid_drive_names()) {
3253         next if !defined($conf->{$ds});
3254
3255         my $drive = parse_drive($ds, $conf->{$ds});
3256         next if !$drive;
3257
3258         &$func($ds, $drive, @param);
3259     }
3260 }
3261
3262 sub foreach_volid {
3263     my ($conf, $func, @param) = @_;
3264
3265     my $volhash = {};
3266
3267     my $test_volid = sub {
3268         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3269
3270         return if !$volid;
3271
3272         $volhash->{$volid}->{cdrom} //= 1;
3273         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3274
3275         $volhash->{$volid}->{replicate} //= 0;
3276         $volhash->{$volid}->{replicate} = 1 if $replicate;
3277
3278         $volhash->{$volid}->{shared} //= 0;
3279         $volhash->{$volid}->{shared} = 1 if $shared;
3280
3281         $volhash->{$volid}->{referenced_in_config} //= 0;
3282         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3283
3284         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3285             if defined($snapname);
3286         $volhash->{$volid}->{size} = $size if $size;
3287     };
3288
3289     foreach_drive($conf, sub {
3290         my ($ds, $drive) = @_;
3291         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3292     });
3293
3294     foreach my $snapname (keys %{$conf->{snapshots}}) {
3295         my $snap = $conf->{snapshots}->{$snapname};
3296         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3297         foreach_drive($snap, sub {
3298             my ($ds, $drive) = @_;
3299             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3300         });
3301     }
3302
3303     foreach my $volid (keys %$volhash) {
3304         &$func($volid, $volhash->{$volid}, @param);
3305     }
3306 }
3307
3308 sub conf_has_serial {
3309     my ($conf) = @_;
3310
3311     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3312         if ($conf->{"serial$i"}) {
3313             return 1;
3314         }
3315     }
3316
3317     return 0;
3318 }
3319
3320 sub conf_has_audio {
3321     my ($conf, $id) = @_;
3322
3323     $id //= 0;
3324     my $audio = $conf->{"audio$id"};
3325     return undef if !defined($audio);
3326
3327     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3328     my $audiodriver = $audioproperties->{driver} // 'spice';
3329
3330     return {
3331         dev => $audioproperties->{device},
3332         dev_id => "audiodev$id",
3333         backend => $audiodriver,
3334         backend_id => "$audiodriver-backend${id}",
3335     };
3336 }
3337
3338 sub vga_conf_has_spice {
3339     my ($vga) = @_;
3340
3341     my $vgaconf = parse_vga($vga);
3342     my $vgatype = $vgaconf->{type};
3343     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3344
3345     return $1 || 1;
3346 }
3347
3348 sub is_native($) {
3349     my ($arch) = @_;
3350     return get_host_arch() eq $arch;
3351 }
3352
3353 sub get_vm_arch {
3354     my ($conf) = @_;
3355     return $conf->{arch} // get_host_arch();
3356 }
3357
3358 my $default_machines = {
3359     x86_64 => 'pc',
3360     aarch64 => 'virt',
3361 };
3362
3363 sub get_vm_machine {
3364     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3365
3366     my $machine = $forcemachine || $conf->{machine};
3367
3368     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3369         $arch //= 'x86_64';
3370         $machine ||= $default_machines->{$arch};
3371         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3372     }
3373
3374     return $machine;
3375 }
3376
3377 sub get_ovmf_files($) {
3378     my ($arch) = @_;
3379
3380     my $ovmf = $OVMF->{$arch}
3381         or die "no OVMF images known for architecture '$arch'\n";
3382
3383     return @$ovmf;
3384 }
3385
3386 my $Arch2Qemu = {
3387     aarch64 => '/usr/bin/qemu-system-aarch64',
3388     x86_64 => '/usr/bin/qemu-system-x86_64',
3389 };
3390 sub get_command_for_arch($) {
3391     my ($arch) = @_;
3392     return '/usr/bin/kvm' if is_native($arch);
3393
3394     my $cmd = $Arch2Qemu->{$arch}
3395         or die "don't know how to emulate architecture '$arch'\n";
3396     return $cmd;
3397 }
3398
3399 sub get_cpu_options {
3400     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3401
3402     my $cpuFlags = [];
3403     my $ostype = $conf->{ostype};
3404
3405     my $cpu = $kvm ? "kvm64" : "qemu64";
3406     if ($arch eq 'aarch64') {
3407         $cpu = 'cortex-a57';
3408     }
3409     my $hv_vendor_id;
3410     if (my $cputype = $conf->{cpu}) {
3411         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3412             or die "Cannot parse cpu description: $cputype\n";
3413         $cpu = $cpuconf->{cputype};
3414         $kvm_off = 1 if $cpuconf->{hidden};
3415         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3416
3417         if (defined(my $flags = $cpuconf->{flags})) {
3418             push @$cpuFlags, split(";", $flags);
3419         }
3420     }
3421
3422     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3423
3424     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3425
3426     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3427
3428     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3429
3430     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3431
3432         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3433         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3434     }
3435
3436     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3437
3438     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3439
3440     push @$cpuFlags, 'kvm=off' if $kvm_off;
3441
3442     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3443         push @$cpuFlags, "vendor=${cpu_vendor}"
3444             if $cpu_vendor ne 'default';
3445     } elsif ($arch ne 'aarch64') {
3446         die "internal error"; # should not happen
3447     }
3448
3449     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3450
3451     return ('-cpu', $cpu);
3452 }
3453
3454 sub config_to_command {
3455     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3456
3457     my $cmd = [];
3458     my $globalFlags = [];
3459     my $machineFlags = [];
3460     my $rtcFlags = [];
3461     my $devices = [];
3462     my $pciaddr = '';
3463     my $bridges = {};
3464     my $vernum = 0; # unknown
3465     my $ostype = $conf->{ostype};
3466     my $winversion = windows_version($ostype);
3467     my $kvm = $conf->{kvm};
3468
3469     my $arch = get_vm_arch($conf);
3470     my $kvm_binary = get_command_for_arch($arch);
3471     my $kvmver = kvm_user_version($kvm_binary);
3472
3473     my $add_pve_version = min_version($kvmver, 4, 1);
3474
3475     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3476     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3477     $kvm //= 1 if is_native($arch);
3478
3479     if ($kvm) {
3480         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3481             if !defined kvm_version();
3482     }
3483
3484     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3485         $vernum = $1*1000000+$2*1000;
3486     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3487         $vernum = $1*1000000+$2*1000+$3;
3488     }
3489
3490     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3491
3492     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3493     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3494     my $use_old_bios_files = undef;
3495     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3496
3497     my $cpuunits = defined($conf->{cpuunits}) ?
3498             $conf->{cpuunits} : $defaults->{cpuunits};
3499
3500     push @$cmd, $kvm_binary;
3501
3502     push @$cmd, '-id', $vmid;
3503
3504     my $vmname = $conf->{name} || "vm$vmid";
3505
3506     push @$cmd, '-name', $vmname;
3507
3508     my $use_virtio = 0;
3509
3510     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3511     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3512     push @$cmd, '-mon', "chardev=qmp,mode=control";
3513
3514     if (min_version($machine_version, 2, 12)) {
3515         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3516         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3517     }
3518
3519     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3520
3521     push @$cmd, '-daemonize';
3522
3523     if ($conf->{smbios1}) {
3524         my $smbios_conf = parse_smbios1($conf->{smbios1});
3525         if ($smbios_conf->{base64}) {
3526             # Do not pass base64 flag to qemu
3527             delete $smbios_conf->{base64};
3528             my $smbios_string = "";
3529             foreach my $key (keys %$smbios_conf) {
3530                 my $value;
3531                 if ($key eq "uuid") {
3532                     $value = $smbios_conf->{uuid}
3533                 } else {
3534                     $value = decode_base64($smbios_conf->{$key});
3535                 }
3536                 # qemu accepts any binary data, only commas need escaping by double comma
3537                 $value =~ s/,/,,/g;
3538                 $smbios_string .= "," . $key . "=" . $value if $value;
3539             }
3540             push @$cmd, '-smbios', "type=1" . $smbios_string;
3541         } else {
3542             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3543         }
3544     }
3545
3546     if ($conf->{vmgenid}) {
3547         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3548     }
3549
3550     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3551     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3552         die "uefi base image not found\n" if ! -f $ovmf_code;
3553
3554         my $path;
3555         my $format;
3556         if (my $efidisk = $conf->{efidisk0}) {
3557             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3558             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3559             $format = $d->{format};
3560             if ($storeid) {
3561                 $path = PVE::Storage::path($storecfg, $d->{file});
3562                 if (!defined($format)) {
3563                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3564                     $format = qemu_img_format($scfg, $volname);
3565                 }
3566             } else {
3567                 $path = $d->{file};
3568                 die "efidisk format must be specified\n"
3569                     if !defined($format);
3570             }
3571         } else {
3572             warn "no efidisk configured! Using temporary efivars disk.\n";
3573             $path = "/tmp/$vmid-ovmf.fd";
3574             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3575             $format = 'raw';
3576         }
3577
3578         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3579         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3580     }
3581
3582     # load q35 config
3583     if ($q35) {
3584         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585         if (min_version($machine_version, 4, 0)) {
3586             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3587         } else {
3588             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3589         }
3590     }
3591
3592     # add usb controllers
3593     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3594     push @$devices, @usbcontrollers if @usbcontrollers;
3595     my $vga = parse_vga($conf->{vga});
3596
3597     my $qxlnum = vga_conf_has_spice($conf->{vga});
3598     $vga->{type} = 'qxl' if $qxlnum;
3599
3600     if (!$vga->{type}) {
3601         if ($arch eq 'aarch64') {
3602             $vga->{type} = 'virtio';
3603         } elsif (min_version($machine_version, 2, 9)) {
3604             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3605         } else {
3606             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3607         }
3608     }
3609
3610     # enable absolute mouse coordinates (needed by vnc)
3611     my $tablet;
3612     if (defined($conf->{tablet})) {
3613         $tablet = $conf->{tablet};
3614     } else {
3615         $tablet = $defaults->{tablet};
3616         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3617         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3618     }
3619
3620     if ($tablet) {
3621         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3622         my $kbd = print_keyboarddevice_full($conf, $arch);
3623         push @$devices, '-device', $kbd if defined($kbd);
3624     }
3625
3626     my $kvm_off = 0;
3627     my $gpu_passthrough;
3628
3629     # host pci devices
3630     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3631         my $id = "hostpci$i";
3632         my $d = parse_hostpci($conf->{$id});
3633         next if !$d;
3634
3635         if (my $pcie = $d->{pcie}) {
3636             die "q35 machine model is not enabled" if !$q35;
3637             # win7 wants to have the pcie devices directly on the pcie bus
3638             # instead of in the root port
3639             if ($winversion == 7) {
3640                 $pciaddr = print_pcie_addr("${id}bus0");
3641             } else {
3642                 # add more root ports if needed, 4 are present by default
3643                 # by pve-q35 cfgs, rest added here on demand.
3644                 if ($i > 3) {
3645                     push @$devices, '-device', print_pcie_root_port($i);
3646                 }
3647                 $pciaddr = print_pcie_addr($id);
3648             }
3649         } else {
3650             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3651         }
3652
3653         my $xvga = '';
3654         if ($d->{'x-vga'}) {
3655             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3656             $kvm_off = 1;
3657             $vga->{type} = 'none' if !defined($conf->{vga});
3658             $gpu_passthrough = 1;
3659         }
3660
3661         my $pcidevices = $d->{pciid};
3662         my $multifunction = 1 if @$pcidevices > 1;
3663
3664         my $sysfspath;
3665         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3666             my $pci_id = $pcidevices->[0]->{id};
3667             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3668             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3669         } elsif ($d->{mdev}) {
3670             warn "ignoring mediated device '$id' with multifunction device\n";
3671         }
3672
3673         my $j=0;
3674         foreach my $pcidevice (@$pcidevices) {
3675             my $devicestr = "vfio-pci";
3676
3677             if ($sysfspath) {
3678                 $devicestr .= ",sysfsdev=$sysfspath";
3679             } else {
3680                 $devicestr .= ",host=$pcidevice->{id}";
3681             }
3682
3683             my $mf_addr = $multifunction ? ".$j" : '';
3684             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3685
3686             if ($j == 0) {
3687                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3688                 $devicestr .= "$xvga";
3689                 $devicestr .= ",multifunction=on" if $multifunction;
3690                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3691             }
3692
3693             push @$devices, '-device', $devicestr;
3694             $j++;
3695         }
3696     }
3697
3698     # usb devices
3699     my $usb_dev_features = {};
3700     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3701
3702     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3703     push @$devices, @usbdevices if @usbdevices;
3704     # serial devices
3705     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3706         if (my $path = $conf->{"serial$i"}) {
3707             if ($path eq 'socket') {
3708                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3709                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3710                 # On aarch64, serial0 is the UART device. Qemu only allows
3711                 # connecting UART devices via the '-serial' command line, as
3712                 # the device has a fixed slot on the hardware...
3713                 if ($arch eq 'aarch64' && $i == 0) {
3714                     push @$devices, '-serial', "chardev:serial$i";
3715                 } else {
3716                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3717                 }
3718             } else {
3719                 die "no such serial device\n" if ! -c $path;
3720                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3721                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3722             }
3723         }
3724     }
3725
3726     # parallel devices
3727     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3728         if (my $path = $conf->{"parallel$i"}) {
3729             die "no such parallel device\n" if ! -c $path;
3730             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3731             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3732             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3733         }
3734     }
3735
3736     if (my $audio = conf_has_audio($conf)) {
3737
3738         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3739
3740         my $id = $audio->{dev_id};
3741         if ($audio->{dev} eq 'AC97') {
3742             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3743         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3744             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3745             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3746             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3747         } else {
3748             die "unkown audio device '$audio->{dev}', implement me!";
3749         }
3750
3751         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3752     }
3753
3754     my $sockets = 1;
3755     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3756     $sockets = $conf->{sockets} if  $conf->{sockets};
3757
3758     my $cores = $conf->{cores} || 1;
3759
3760     my $maxcpus = $sockets * $cores;
3761
3762     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3763
3764     my $allowed_vcpus = $cpuinfo->{cpus};
3765
3766     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3767         if ($allowed_vcpus < $maxcpus);
3768
3769     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3770
3771         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3772         for (my $i = 2; $i <= $vcpus; $i++)  {
3773             my $cpustr = print_cpu_device($conf,$i);
3774             push @$cmd, '-device', $cpustr;
3775         }
3776
3777     } else {
3778
3779         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3780     }
3781     push @$cmd, '-nodefaults';
3782
3783     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3784
3785     my $bootindex_hash = {};
3786     my $i = 1;
3787     foreach my $o (split(//, $bootorder)) {
3788         $bootindex_hash->{$o} = $i*100;
3789         $i++;
3790     }
3791
3792     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3793
3794     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3795
3796     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3797
3798     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3799         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3800         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3801         push @$cmd,  '-vnc', "unix:$socket,password";
3802     } else {
3803         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3804         push @$cmd, '-nographic';
3805     }
3806
3807     # time drift fix
3808     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3809
3810     my $useLocaltime = $conf->{localtime};
3811
3812     if ($winversion >= 5) { # windows
3813         $useLocaltime = 1 if !defined($conf->{localtime});
3814
3815         # use time drift fix when acpi is enabled
3816         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3817             $tdf = 1 if !defined($conf->{tdf});
3818         }
3819     }
3820
3821     if ($winversion >= 6) {
3822         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3823         push @$cmd, '-no-hpet';
3824     }
3825
3826     push @$rtcFlags, 'driftfix=slew' if $tdf;
3827
3828     if (!$kvm) {
3829         push @$machineFlags, 'accel=tcg';
3830     }
3831
3832     if ($machine_type) {
3833         push @$machineFlags, "type=${machine_type}";
3834     }
3835
3836     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3837         push @$rtcFlags, "base=$conf->{startdate}";
3838     } elsif ($useLocaltime) {
3839         push @$rtcFlags, 'base=localtime';
3840     }
3841
3842     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3843
3844     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3845
3846     push @$cmd, '-S' if $conf->{freeze};
3847
3848     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3849
3850     my $guest_agent = parse_guest_agent($conf);
3851
3852     if ($guest_agent->{enabled}) {
3853         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3854         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3855
3856         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3857             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3858             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3859             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3860         } elsif ($guest_agent->{type} eq 'isa') {
3861             push @$devices, '-device', "isa-serial,chardev=qga0";
3862         }
3863     }
3864
3865     my $spice_port;
3866
3867     if ($qxlnum) {
3868         if ($qxlnum > 1) {
3869             if ($winversion){
3870                 for(my $i = 1; $i < $qxlnum; $i++){
3871                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3872                 }
3873             } else {
3874                 # assume other OS works like Linux
3875                 my ($ram, $vram) = ("134217728", "67108864");
3876                 if ($vga->{memory}) {
3877                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3878                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3879                 }
3880                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3881                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3882             }
3883         }
3884
3885         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3886
3887         my $nodename = PVE::INotify::nodename();
3888         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3889         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3890         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3891
3892         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3893         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3894         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3895
3896         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3897         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3898
3899         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3900         if ($spice_enhancement->{foldersharing}) {
3901             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3902             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3903         }
3904
3905         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3906         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3907         push @$devices, '-spice', "$spice_opts";
3908     }
3909
3910     # enable balloon by default, unless explicitly disabled
3911     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3912         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3913         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3914     }
3915
3916     if ($conf->{watchdog}) {
3917         my $wdopts = parse_watchdog($conf->{watchdog});
3918         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3919         my $watchdog = $wdopts->{model} || 'i6300esb';
3920         push @$devices, '-device', "$watchdog$pciaddr";
3921         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3922     }
3923
3924     my $vollist = [];
3925     my $scsicontroller = {};
3926     my $ahcicontroller = {};
3927     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3928
3929     # Add iscsi initiator name if available
3930     if (my $initiator = get_initiator_name()) {
3931         push @$devices, '-iscsi', "initiator-name=$initiator";
3932     }
3933
3934     foreach_drive($conf, sub {
3935         my ($ds, $drive) = @_;
3936
3937         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3938             push @$vollist, $drive->{file};
3939         }
3940
3941         # ignore efidisk here, already added in bios/fw handling code above
3942         return if $drive->{interface} eq 'efidisk';
3943
3944         $use_virtio = 1 if $ds =~ m/^virtio/;
3945
3946         if (drive_is_cdrom ($drive)) {
3947             if ($bootindex_hash->{d}) {
3948                 $drive->{bootindex} = $bootindex_hash->{d};
3949                 $bootindex_hash->{d} += 1;
3950             }
3951         } else {
3952             if ($bootindex_hash->{c}) {
3953                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3954                 $bootindex_hash->{c} += 1;
3955             }
3956         }
3957
3958         if($drive->{interface} eq 'virtio'){
3959            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3960         }
3961
3962         if ($drive->{interface} eq 'scsi') {
3963
3964             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3965
3966             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3967             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3968
3969             my $iothread = '';
3970             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3971                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3972                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3973             } elsif ($drive->{iothread}) {
3974                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3975             }
3976
3977             my $queues = '';
3978             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3979                 $queues = ",num_queues=$drive->{queues}";
3980             }
3981
3982             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3983             $scsicontroller->{$controller}=1;
3984         }
3985
3986         if ($drive->{interface} eq 'sata') {
3987            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3988            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3989            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3990            $ahcicontroller->{$controller}=1;
3991         }
3992
3993         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3994         push @$devices, '-drive',$drive_cmd;
3995         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3996     });
3997
3998     for (my $i = 0; $i < $MAX_NETS; $i++) {
3999          next if !$conf->{"net$i"};
4000          my $d = parse_net($conf->{"net$i"});
4001          next if !$d;
4002
4003          $use_virtio = 1 if $d->{model} eq 'virtio';
4004
4005          if ($bootindex_hash->{n}) {
4006             $d->{bootindex} = $bootindex_hash->{n};
4007             $bootindex_hash->{n} += 1;
4008          }
4009
4010          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4011          push @$devices, '-netdev', $netdevfull;
4012
4013          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4014          push @$devices, '-device', $netdevicefull;
4015     }
4016
4017     if ($conf->{ivshmem}) {
4018         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4019
4020         my $bus;
4021         if ($q35) {
4022             $bus = print_pcie_addr("ivshmem");
4023         } else {
4024             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4025         }
4026
4027         my $ivshmem_name = $ivshmem->{name} // $vmid;
4028         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4029
4030         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4031         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4032     }
4033
4034     if (!$q35) {
4035         # add pci bridges
4036         if (min_version($machine_version, 2, 3)) {
4037            $bridges->{1} = 1;
4038            $bridges->{2} = 1;
4039         }
4040
4041         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4042
4043         for my $k (sort {$b cmp $a} keys %$bridges) {
4044             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4045             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4046         }
4047     }
4048
4049     push @$cmd, @$devices;
4050     push @$cmd, '-rtc', join(',', @$rtcFlags)
4051         if scalar(@$rtcFlags);
4052     push @$cmd, '-machine', join(',', @$machineFlags)
4053         if scalar(@$machineFlags);
4054     push @$cmd, '-global', join(',', @$globalFlags)
4055         if scalar(@$globalFlags);
4056
4057     if (my $vmstate = $conf->{vmstate}) {
4058         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4059         push @$vollist, $vmstate;
4060         push @$cmd, '-loadstate', $statepath;
4061         print "activating and using '$vmstate' as vmstate\n";
4062     }
4063
4064     # add custom args
4065     if ($conf->{args}) {
4066         my $aa = PVE::Tools::split_args($conf->{args});
4067         push @$cmd, @$aa;
4068     }
4069
4070     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4071 }
4072
4073 sub spice_port {
4074     my ($vmid) = @_;
4075
4076     my $res = mon_cmd($vmid, 'query-spice');
4077
4078     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4079 }
4080
4081 sub vm_devices_list {
4082     my ($vmid) = @_;
4083
4084     my $res = mon_cmd($vmid, 'query-pci');
4085     my $devices_to_check = [];
4086     my $devices = {};
4087     foreach my $pcibus (@$res) {
4088         push @$devices_to_check, @{$pcibus->{devices}},
4089     }
4090
4091     while (@$devices_to_check) {
4092         my $to_check = [];
4093         for my $d (@$devices_to_check) {
4094             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4095             next if !$d->{'pci_bridge'};
4096
4097             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4098             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4099         }
4100         $devices_to_check = $to_check;
4101     }
4102
4103     my $resblock = mon_cmd($vmid, 'query-block');
4104     foreach my $block (@$resblock) {
4105         if($block->{device} =~ m/^drive-(\S+)/){
4106                 $devices->{$1} = 1;
4107         }
4108     }
4109
4110     my $resmice = mon_cmd($vmid, 'query-mice');
4111     foreach my $mice (@$resmice) {
4112         if ($mice->{name} eq 'QEMU HID Tablet') {
4113             $devices->{tablet} = 1;
4114             last;
4115         }
4116     }
4117
4118     # for usb devices there is no query-usb
4119     # but we can iterate over the entries in
4120     # qom-list path=/machine/peripheral
4121     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4122     foreach my $per (@$resperipheral) {
4123         if ($per->{name} =~ m/^usb\d+$/) {
4124             $devices->{$per->{name}} = 1;
4125         }
4126     }
4127
4128     return $devices;
4129 }
4130
4131 sub vm_deviceplug {
4132     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4133
4134     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4135
4136     my $devices_list = vm_devices_list($vmid);
4137     return 1 if defined($devices_list->{$deviceid});
4138
4139     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4140
4141     if ($deviceid eq 'tablet') {
4142
4143         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4144
4145     } elsif ($deviceid eq 'keyboard') {
4146
4147         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4148
4149     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4150
4151         die "usb hotplug currently not reliable\n";
4152         # since we can't reliably hot unplug all added usb devices
4153         # and usb passthrough disables live migration
4154         # we disable usb hotplugging for now
4155         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4156
4157     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4158
4159         qemu_iothread_add($vmid, $deviceid, $device);
4160
4161         qemu_driveadd($storecfg, $vmid, $device);
4162         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4163
4164         qemu_deviceadd($vmid, $devicefull);
4165         eval { qemu_deviceaddverify($vmid, $deviceid); };
4166         if (my $err = $@) {
4167             eval { qemu_drivedel($vmid, $deviceid); };
4168             warn $@ if $@;
4169             die $err;
4170         }
4171
4172     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4173
4174
4175         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4176         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4177         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4178
4179         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4180
4181         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4182             qemu_iothread_add($vmid, $deviceid, $device);
4183             $devicefull .= ",iothread=iothread-$deviceid";
4184         }
4185
4186         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4187             $devicefull .= ",num_queues=$device->{queues}";
4188         }
4189
4190         qemu_deviceadd($vmid, $devicefull);
4191         qemu_deviceaddverify($vmid, $deviceid);
4192
4193     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4194
4195         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4196         qemu_driveadd($storecfg, $vmid, $device);
4197
4198         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4199         eval { qemu_deviceadd($vmid, $devicefull); };
4200         if (my $err = $@) {
4201             eval { qemu_drivedel($vmid, $deviceid); };
4202             warn $@ if $@;
4203             die $err;
4204         }
4205
4206     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4207
4208         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4209
4210         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4211         my $use_old_bios_files = undef;
4212         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4213
4214         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4215         qemu_deviceadd($vmid, $netdevicefull);
4216         eval {
4217             qemu_deviceaddverify($vmid, $deviceid);
4218             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4219         };
4220         if (my $err = $@) {
4221             eval { qemu_netdevdel($vmid, $deviceid); };
4222             warn $@ if $@;
4223             die $err;
4224         }
4225
4226     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4227
4228         my $bridgeid = $2;
4229         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4230         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4231
4232         qemu_deviceadd($vmid, $devicefull);
4233         qemu_deviceaddverify($vmid, $deviceid);
4234
4235     } else {
4236         die "can't hotplug device '$deviceid'\n";
4237     }
4238
4239     return 1;
4240 }
4241
4242 # fixme: this should raise exceptions on error!
4243 sub vm_deviceunplug {
4244     my ($vmid, $conf, $deviceid) = @_;
4245
4246     my $devices_list = vm_devices_list($vmid);
4247     return 1 if !defined($devices_list->{$deviceid});
4248
4249     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4250
4251     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4252
4253         qemu_devicedel($vmid, $deviceid);
4254
4255     } elsif ($deviceid =~ m/^usb\d+$/) {
4256
4257         die "usb hotplug currently not reliable\n";
4258         # when unplugging usb devices this way,
4259         # there may be remaining usb controllers/hubs
4260         # so we disable it for now
4261         qemu_devicedel($vmid, $deviceid);
4262         qemu_devicedelverify($vmid, $deviceid);
4263
4264     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4265
4266         qemu_devicedel($vmid, $deviceid);
4267         qemu_devicedelverify($vmid, $deviceid);
4268         qemu_drivedel($vmid, $deviceid);
4269         qemu_iothread_del($conf, $vmid, $deviceid);
4270
4271     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4272
4273         qemu_devicedel($vmid, $deviceid);
4274         qemu_devicedelverify($vmid, $deviceid);
4275         qemu_iothread_del($conf, $vmid, $deviceid);
4276
4277     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4278
4279         qemu_devicedel($vmid, $deviceid);
4280         qemu_drivedel($vmid, $deviceid);
4281         qemu_deletescsihw($conf, $vmid, $deviceid);
4282
4283     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4284
4285         qemu_devicedel($vmid, $deviceid);
4286         qemu_devicedelverify($vmid, $deviceid);
4287         qemu_netdevdel($vmid, $deviceid);
4288
4289     } else {
4290         die "can't unplug device '$deviceid'\n";
4291     }
4292
4293     return 1;
4294 }
4295
4296 sub qemu_deviceadd {
4297     my ($vmid, $devicefull) = @_;
4298
4299     $devicefull = "driver=".$devicefull;
4300     my %options =  split(/[=,]/, $devicefull);
4301
4302     mon_cmd($vmid, "device_add" , %options);
4303 }
4304
4305 sub qemu_devicedel {
4306     my ($vmid, $deviceid) = @_;
4307
4308     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4309 }
4310
4311 sub qemu_iothread_add {
4312     my($vmid, $deviceid, $device) = @_;
4313
4314     if ($device->{iothread}) {
4315         my $iothreads = vm_iothreads_list($vmid);
4316         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4317     }
4318 }
4319
4320 sub qemu_iothread_del {
4321     my($conf, $vmid, $deviceid) = @_;
4322
4323     my $confid = $deviceid;
4324     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4325         $confid = 'scsi' . $1;
4326     }
4327     my $device = parse_drive($confid, $conf->{$confid});
4328     if ($device->{iothread}) {
4329         my $iothreads = vm_iothreads_list($vmid);
4330         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4331     }
4332 }
4333
4334 sub qemu_objectadd {
4335     my($vmid, $objectid, $qomtype) = @_;
4336
4337     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4338
4339     return 1;
4340 }
4341
4342 sub qemu_objectdel {
4343     my($vmid, $objectid) = @_;
4344
4345     mon_cmd($vmid, "object-del", id => $objectid);
4346
4347     return 1;
4348 }
4349
4350 sub qemu_driveadd {
4351     my ($storecfg, $vmid, $device) = @_;
4352
4353     my $drive = print_drive_full($storecfg, $vmid, $device);
4354     $drive =~ s/\\/\\\\/g;
4355     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4356
4357     # If the command succeeds qemu prints: "OK"
4358     return 1 if $ret =~ m/OK/s;
4359
4360     die "adding drive failed: $ret\n";
4361 }
4362
4363 sub qemu_drivedel {
4364     my($vmid, $deviceid) = @_;
4365
4366     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4367     $ret =~ s/^\s+//;
4368
4369     return 1 if $ret eq "";
4370
4371     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4372     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4373
4374     die "deleting drive $deviceid failed : $ret\n";
4375 }
4376
4377 sub qemu_deviceaddverify {
4378     my ($vmid, $deviceid) = @_;
4379
4380     for (my $i = 0; $i <= 5; $i++) {
4381          my $devices_list = vm_devices_list($vmid);
4382          return 1 if defined($devices_list->{$deviceid});
4383          sleep 1;
4384     }
4385
4386     die "error on hotplug device '$deviceid'\n";
4387 }
4388
4389
4390 sub qemu_devicedelverify {
4391     my ($vmid, $deviceid) = @_;
4392
4393     # need to verify that the device is correctly removed as device_del
4394     # is async and empty return is not reliable
4395
4396     for (my $i = 0; $i <= 5; $i++) {
4397          my $devices_list = vm_devices_list($vmid);
4398          return 1 if !defined($devices_list->{$deviceid});
4399          sleep 1;
4400     }
4401
4402     die "error on hot-unplugging device '$deviceid'\n";
4403 }
4404
4405 sub qemu_findorcreatescsihw {
4406     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4407
4408     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4409
4410     my $scsihwid="$controller_prefix$controller";
4411     my $devices_list = vm_devices_list($vmid);
4412
4413     if(!defined($devices_list->{$scsihwid})) {
4414         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4415     }
4416
4417     return 1;
4418 }
4419
4420 sub qemu_deletescsihw {
4421     my ($conf, $vmid, $opt) = @_;
4422
4423     my $device = parse_drive($opt, $conf->{$opt});
4424
4425     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4426         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4427         return 1;
4428     }
4429
4430     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4431
4432     my $devices_list = vm_devices_list($vmid);
4433     foreach my $opt (keys %{$devices_list}) {
4434         if (PVE::QemuServer::is_valid_drivename($opt)) {
4435             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4436             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4437                 return 1;
4438             }
4439         }
4440     }
4441
4442     my $scsihwid="scsihw$controller";
4443
4444     vm_deviceunplug($vmid, $conf, $scsihwid);
4445
4446     return 1;
4447 }
4448
4449 sub qemu_add_pci_bridge {
4450     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4451
4452     my $bridges = {};
4453
4454     my $bridgeid;
4455
4456     print_pci_addr($device, $bridges, $arch, $machine_type);
4457
4458     while (my ($k, $v) = each %$bridges) {
4459         $bridgeid = $k;
4460     }
4461     return 1 if !defined($bridgeid) || $bridgeid < 1;
4462
4463     my $bridge = "pci.$bridgeid";
4464     my $devices_list = vm_devices_list($vmid);
4465
4466     if (!defined($devices_list->{$bridge})) {
4467         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4468     }
4469
4470     return 1;
4471 }
4472
4473 sub qemu_set_link_status {
4474     my ($vmid, $device, $up) = @_;
4475
4476     mon_cmd($vmid, "set_link", name => $device,
4477                up => $up ? JSON::true : JSON::false);
4478 }
4479
4480 sub qemu_netdevadd {
4481     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4482
4483     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4484     my %options =  split(/[=,]/, $netdev);
4485
4486     mon_cmd($vmid, "netdev_add",  %options);
4487     return 1;
4488 }
4489
4490 sub qemu_netdevdel {
4491     my ($vmid, $deviceid) = @_;
4492
4493     mon_cmd($vmid, "netdev_del", id => $deviceid);
4494 }
4495
4496 sub qemu_usb_hotplug {
4497     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4498
4499     return if !$device;
4500
4501     # remove the old one first
4502     vm_deviceunplug($vmid, $conf, $deviceid);
4503
4504     # check if xhci controller is necessary and available
4505     if ($device->{usb3}) {
4506
4507         my $devicelist = vm_devices_list($vmid);
4508
4509         if (!$devicelist->{xhci}) {
4510             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4511             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4512         }
4513     }
4514     my $d = parse_usb_device($device->{host});
4515     $d->{usb3} = $device->{usb3};
4516
4517     # add the new one
4518     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4519 }
4520
4521 sub qemu_cpu_hotplug {
4522     my ($vmid, $conf, $vcpus) = @_;
4523
4524     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4525
4526     my $sockets = 1;
4527     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4528     $sockets = $conf->{sockets} if  $conf->{sockets};
4529     my $cores = $conf->{cores} || 1;
4530     my $maxcpus = $sockets * $cores;
4531
4532     $vcpus = $maxcpus if !$vcpus;
4533
4534     die "you can't add more vcpus than maxcpus\n"
4535         if $vcpus > $maxcpus;
4536
4537     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4538
4539     if ($vcpus < $currentvcpus) {
4540
4541         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4542
4543             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4544                 qemu_devicedel($vmid, "cpu$i");
4545                 my $retry = 0;
4546                 my $currentrunningvcpus = undef;
4547                 while (1) {
4548                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4549                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4550                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4551                     $retry++;
4552                     sleep 1;
4553                 }
4554                 #update conf after each succesfull cpu unplug
4555                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4556                 PVE::QemuConfig->write_config($vmid, $conf);
4557             }
4558         } else {
4559             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4560         }
4561
4562         return;
4563     }
4564
4565     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4566     die "vcpus in running vm does not match its configuration\n"
4567         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4568
4569     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4570
4571         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4572             my $cpustr = print_cpu_device($conf, $i);
4573             qemu_deviceadd($vmid, $cpustr);
4574
4575             my $retry = 0;
4576             my $currentrunningvcpus = undef;
4577             while (1) {
4578                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4579                 last if scalar(@{$currentrunningvcpus}) == $i;
4580                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4581                 sleep 1;
4582                 $retry++;
4583             }
4584             #update conf after each succesfull cpu hotplug
4585             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4586             PVE::QemuConfig->write_config($vmid, $conf);
4587         }
4588     } else {
4589
4590         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4591             mon_cmd($vmid, "cpu-add", id => int($i));
4592         }
4593     }
4594 }
4595
4596 sub qemu_block_set_io_throttle {
4597     my ($vmid, $deviceid,
4598         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4599         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4600         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4601         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4602
4603     return if !check_running($vmid) ;
4604
4605     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4606         bps => int($bps),
4607         bps_rd => int($bps_rd),
4608         bps_wr => int($bps_wr),
4609         iops => int($iops),
4610         iops_rd => int($iops_rd),
4611         iops_wr => int($iops_wr),
4612         bps_max => int($bps_max),
4613         bps_rd_max => int($bps_rd_max),
4614         bps_wr_max => int($bps_wr_max),
4615         iops_max => int($iops_max),
4616         iops_rd_max => int($iops_rd_max),
4617         iops_wr_max => int($iops_wr_max),
4618         bps_max_length => int($bps_max_length),
4619         bps_rd_max_length => int($bps_rd_max_length),
4620         bps_wr_max_length => int($bps_wr_max_length),
4621         iops_max_length => int($iops_max_length),
4622         iops_rd_max_length => int($iops_rd_max_length),
4623         iops_wr_max_length => int($iops_wr_max_length),
4624     );
4625
4626 }
4627
4628 # old code, only used to shutdown old VM after update
4629 sub __read_avail {
4630     my ($fh, $timeout) = @_;
4631
4632     my $sel = new IO::Select;
4633     $sel->add($fh);
4634
4635     my $res = '';
4636     my $buf;
4637
4638     my @ready;
4639     while (scalar (@ready = $sel->can_read($timeout))) {
4640         my $count;
4641         if ($count = $fh->sysread($buf, 8192)) {
4642             if ($buf =~ /^(.*)\(qemu\) $/s) {
4643                 $res .= $1;
4644                 last;
4645             } else {
4646                 $res .= $buf;
4647             }
4648         } else {
4649             if (!defined($count)) {
4650                 die "$!\n";
4651             }
4652             last;
4653         }
4654     }
4655
4656     die "monitor read timeout\n" if !scalar(@ready);
4657
4658     return $res;
4659 }
4660
4661 sub qemu_block_resize {
4662     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4663
4664     my $running = check_running($vmid);
4665
4666     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4667
4668     return if !$running;
4669
4670     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4671
4672 }
4673
4674 sub qemu_volume_snapshot {
4675     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4676
4677     my $running = check_running($vmid);
4678
4679     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4680         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4681     } else {
4682         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4683     }
4684 }
4685
4686 sub qemu_volume_snapshot_delete {
4687     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4688
4689     my $running = check_running($vmid);
4690
4691     if($running) {
4692
4693         $running = undef;
4694         my $conf = PVE::QemuConfig->load_config($vmid);
4695         foreach_drive($conf, sub {
4696             my ($ds, $drive) = @_;
4697             $running = 1 if $drive->{file} eq $volid;
4698         });
4699     }
4700
4701     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4702         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4703     } else {
4704         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4705     }
4706 }
4707
4708 sub set_migration_caps {
4709     my ($vmid) = @_;
4710
4711     my $cap_ref = [];
4712
4713     my $enabled_cap = {
4714         "auto-converge" => 1,
4715         "xbzrle" => 1,
4716         "x-rdma-pin-all" => 0,
4717         "zero-blocks" => 0,
4718         "compress" => 0
4719     };
4720
4721     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4722
4723     for my $supported_capability (@$supported_capabilities) {
4724         push @$cap_ref, {
4725             capability => $supported_capability->{capability},
4726             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4727         };
4728     }
4729
4730     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4731 }
4732
4733 my $fast_plug_option = {
4734     'lock' => 1,
4735     'name' => 1,
4736     'onboot' => 1,
4737     'shares' => 1,
4738     'startup' => 1,
4739     'description' => 1,
4740     'protection' => 1,
4741     'vmstatestorage' => 1,
4742     'hookscript' => 1,
4743     'tags' => 1,
4744 };
4745
4746 # hotplug changes in [PENDING]
4747 # $selection hash can be used to only apply specified options, for
4748 # example: { cores => 1 } (only apply changed 'cores')
4749 # $errors ref is used to return error messages
4750 sub vmconfig_hotplug_pending {
4751     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4752
4753     my $defaults = load_defaults();
4754     my $arch = get_vm_arch($conf);
4755     my $machine_type = get_vm_machine($conf, undef, $arch);
4756
4757     # commit values which do not have any impact on running VM first
4758     # Note: those option cannot raise errors, we we do not care about
4759     # $selection and always apply them.
4760
4761     my $add_error = sub {
4762         my ($opt, $msg) = @_;
4763         $errors->{$opt} = "hotplug problem - $msg";
4764     };
4765
4766     my $changes = 0;
4767     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4768         if ($fast_plug_option->{$opt}) {
4769             $conf->{$opt} = $conf->{pending}->{$opt};
4770             delete $conf->{pending}->{$opt};
4771             $changes = 1;
4772         }
4773     }
4774
4775     if ($changes) {
4776         PVE::QemuConfig->write_config($vmid, $conf);
4777         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4778     }
4779
4780     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4781
4782     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4783     foreach my $opt (sort keys %$pending_delete_hash) {
4784         next if $selection && !$selection->{$opt};
4785         my $force = $pending_delete_hash->{$opt}->{force};
4786         eval {
4787             if ($opt eq 'hotplug') {
4788                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4789             } elsif ($opt eq 'tablet') {
4790                 die "skip\n" if !$hotplug_features->{usb};
4791                 if ($defaults->{tablet}) {
4792                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4793                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4794                         if $arch eq 'aarch64';
4795                 } else {
4796                     vm_deviceunplug($vmid, $conf, 'tablet');
4797                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4798                 }
4799             } elsif ($opt =~ m/^usb\d+/) {
4800                 die "skip\n";
4801                 # since we cannot reliably hot unplug usb devices
4802                 # we are disabling it
4803                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4804                 vm_deviceunplug($vmid, $conf, $opt);
4805             } elsif ($opt eq 'vcpus') {
4806                 die "skip\n" if !$hotplug_features->{cpu};
4807                 qemu_cpu_hotplug($vmid, $conf, undef);
4808             } elsif ($opt eq 'balloon') {
4809                 # enable balloon device is not hotpluggable
4810                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4811                 # here we reset the ballooning value to memory
4812                 my $balloon = $conf->{memory} || $defaults->{memory};
4813                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4814             } elsif ($fast_plug_option->{$opt}) {
4815                 # do nothing
4816             } elsif ($opt =~ m/^net(\d+)$/) {
4817                 die "skip\n" if !$hotplug_features->{network};
4818                 vm_deviceunplug($vmid, $conf, $opt);
4819             } elsif (is_valid_drivename($opt)) {
4820                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4821                 vm_deviceunplug($vmid, $conf, $opt);
4822                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4823             } elsif ($opt =~ m/^memory$/) {
4824                 die "skip\n" if !$hotplug_features->{memory};
4825                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4826             } elsif ($opt eq 'cpuunits') {
4827                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4828             } elsif ($opt eq 'cpulimit') {
4829                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4830             } else {
4831                 die "skip\n";
4832             }
4833         };
4834         if (my $err = $@) {
4835             &$add_error($opt, $err) if $err ne "skip\n";
4836         } else {
4837             # save new config if hotplug was successful
4838             delete $conf->{$opt};
4839             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4840             PVE::QemuConfig->write_config($vmid, $conf);
4841             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4842         }
4843     }
4844
4845     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4846     $apply_pending_cloudinit = sub {
4847         return if $apply_pending_cloudinit_done; # once is enough
4848         $apply_pending_cloudinit_done = 1; # once is enough
4849
4850         my ($key, $value) = @_;
4851
4852         my @cloudinit_opts = keys %$confdesc_cloudinit;
4853         foreach my $opt (keys %{$conf->{pending}}) {
4854             next if !grep { $_ eq $opt } @cloudinit_opts;
4855             $conf->{$opt} = delete $conf->{pending}->{$opt};
4856         }
4857
4858         my $new_conf = { %$conf };
4859         $new_conf->{$key} = $value;
4860         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4861     };
4862
4863     foreach my $opt (keys %{$conf->{pending}}) {
4864         next if $selection && !$selection->{$opt};
4865         my $value = $conf->{pending}->{$opt};
4866         eval {
4867             if ($opt eq 'hotplug') {
4868                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4869             } elsif ($opt eq 'tablet') {
4870                 die "skip\n" if !$hotplug_features->{usb};
4871                 if ($value == 1) {
4872                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4873                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4874                         if $arch eq 'aarch64';
4875                 } elsif ($value == 0) {
4876                     vm_deviceunplug($vmid, $conf, 'tablet');
4877                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4878                 }
4879             } elsif ($opt =~ m/^usb\d+$/) {
4880                 die "skip\n";
4881                 # since we cannot reliably hot unplug usb devices
4882                 # we are disabling it
4883                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4884                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4885                 die "skip\n" if !$d;
4886                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4887             } elsif ($opt eq 'vcpus') {
4888                 die "skip\n" if !$hotplug_features->{cpu};
4889                 qemu_cpu_hotplug($vmid, $conf, $value);
4890             } elsif ($opt eq 'balloon') {
4891                 # enable/disable balloning device is not hotpluggable
4892                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4893                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4894                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4895
4896                 # allow manual ballooning if shares is set to zero
4897                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4898                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4899                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4900                 }
4901             } elsif ($opt =~ m/^net(\d+)$/) {
4902                 # some changes can be done without hotplug
4903                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4904                                     $vmid, $opt, $value, $arch, $machine_type);
4905             } elsif (is_valid_drivename($opt)) {
4906                 die "skip\n" if $opt eq 'efidisk0';
4907                 # some changes can be done without hotplug
4908                 my $drive = parse_drive($opt, $value);
4909                 if (drive_is_cloudinit($drive)) {
4910                     &$apply_pending_cloudinit($opt, $value);
4911                 }
4912                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4913                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4914             } elsif ($opt =~ m/^memory$/) { #dimms
4915                 die "skip\n" if !$hotplug_features->{memory};
4916                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4917             } elsif ($opt eq 'cpuunits') {
4918                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4919             } elsif ($opt eq 'cpulimit') {
4920                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4921                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4922             } else {
4923                 die "skip\n";  # skip non-hot-pluggable options
4924             }
4925         };
4926         if (my $err = $@) {
4927             &$add_error($opt, $err) if $err ne "skip\n";
4928         } else {
4929             # save new config if hotplug was successful
4930             $conf->{$opt} = $value;
4931             delete $conf->{pending}->{$opt};
4932             PVE::QemuConfig->write_config($vmid, $conf);
4933             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4934         }
4935     }
4936 }
4937
4938 sub try_deallocate_drive {
4939     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4940
4941     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4942         my $volid = $drive->{file};
4943         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4944             my $sid = PVE::Storage::parse_volume_id($volid);
4945             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4946
4947             # check if the disk is really unused
4948             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4949                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4950             PVE::Storage::vdisk_free($storecfg, $volid);
4951             return 1;
4952         } else {
4953             # If vm is not owner of this disk remove from config
4954             return 1;
4955         }
4956     }
4957
4958     return undef;
4959 }
4960
4961 sub vmconfig_delete_or_detach_drive {
4962     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4963
4964     my $drive = parse_drive($opt, $conf->{$opt});
4965
4966     my $rpcenv = PVE::RPCEnvironment::get();
4967     my $authuser = $rpcenv->get_user();
4968
4969     if ($force) {
4970         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4971         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4972     } else {
4973         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4974     }
4975 }
4976
4977
4978
4979 sub vmconfig_apply_pending {
4980     my ($vmid, $conf, $storecfg) = @_;
4981
4982     # cold plug
4983
4984     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4985     foreach my $opt (sort keys %$pending_delete_hash) {
4986         die "internal error" if $opt =~ m/^unused/;
4987         my $force = $pending_delete_hash->{$opt}->{force};
4988         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4989         if (!defined($conf->{$opt})) {
4990             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4991             PVE::QemuConfig->write_config($vmid, $conf);
4992         } elsif (is_valid_drivename($opt)) {
4993             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4994             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4995             delete $conf->{$opt};
4996             PVE::QemuConfig->write_config($vmid, $conf);
4997         } else {
4998             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4999             delete $conf->{$opt};
5000             PVE::QemuConfig->write_config($vmid, $conf);
5001         }
5002     }
5003
5004     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5005
5006     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5007         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5008
5009         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5010             # skip if nothing changed
5011         } elsif (is_valid_drivename($opt)) {
5012             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5013                 if defined($conf->{$opt});
5014             $conf->{$opt} = $conf->{pending}->{$opt};
5015         } else {
5016             $conf->{$opt} = $conf->{pending}->{$opt};
5017         }
5018
5019         delete $conf->{pending}->{$opt};
5020         PVE::QemuConfig->write_config($vmid, $conf);
5021     }
5022 }
5023
5024 my $safe_num_ne = sub {
5025     my ($a, $b) = @_;
5026
5027     return 0 if !defined($a) && !defined($b);
5028     return 1 if !defined($a);
5029     return 1 if !defined($b);
5030
5031     return $a != $b;
5032 };
5033
5034 my $safe_string_ne = sub {
5035     my ($a, $b) = @_;
5036
5037     return 0 if !defined($a) && !defined($b);
5038     return 1 if !defined($a);
5039     return 1 if !defined($b);
5040
5041     return $a ne $b;
5042 };
5043
5044 sub vmconfig_update_net {
5045     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5046
5047     my $newnet = parse_net($value);
5048
5049     if ($conf->{$opt}) {
5050         my $oldnet = parse_net($conf->{$opt});
5051
5052         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5053             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5054             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5055             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5056
5057             # for non online change, we try to hot-unplug
5058             die "skip\n" if !$hotplug;
5059             vm_deviceunplug($vmid, $conf, $opt);
5060         } else {
5061
5062             die "internal error" if $opt !~ m/net(\d+)/;
5063             my $iface = "tap${vmid}i$1";
5064
5065             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5066                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5067                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5068                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5069                 PVE::Network::tap_unplug($iface);
5070                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5071             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5072                 # Rate can be applied on its own but any change above needs to
5073                 # include the rate in tap_plug since OVS resets everything.
5074                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5075             }
5076
5077             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5078                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5079             }
5080
5081             return 1;
5082         }
5083     }
5084
5085     if ($hotplug) {
5086         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5087     } else {
5088         die "skip\n";
5089     }
5090 }
5091
5092 sub vmconfig_update_disk {
5093     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5094
5095     # fixme: do we need force?
5096
5097     my $drive = parse_drive($opt, $value);
5098
5099     if ($conf->{$opt}) {
5100
5101         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5102
5103             my $media = $drive->{media} || 'disk';
5104             my $oldmedia = $old_drive->{media} || 'disk';
5105             die "unable to change media type\n" if $media ne $oldmedia;
5106
5107             if (!drive_is_cdrom($old_drive)) {
5108
5109                 if ($drive->{file} ne $old_drive->{file}) {
5110
5111                     die "skip\n" if !$hotplug;
5112
5113                     # unplug and register as unused
5114                     vm_deviceunplug($vmid, $conf, $opt);
5115                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5116
5117                 } else {
5118                     # update existing disk
5119
5120                     # skip non hotpluggable value
5121                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5122                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5123                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5124                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5125                         die "skip\n";
5126                     }
5127
5128                     # apply throttle
5129                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5130                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5131                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5132                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5133                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5134                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5135                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5136                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5137                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5138                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5139                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5140                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5141                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5142                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5143                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5144                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5145                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5146                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5147
5148                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5149                                                    ($drive->{mbps} || 0)*1024*1024,
5150                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5151                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5152                                                    $drive->{iops} || 0,
5153                                                    $drive->{iops_rd} || 0,
5154                                                    $drive->{iops_wr} || 0,
5155                                                    ($drive->{mbps_max} || 0)*1024*1024,
5156                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5157                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5158                                                    $drive->{iops_max} || 0,
5159                                                    $drive->{iops_rd_max} || 0,
5160                                                    $drive->{iops_wr_max} || 0,
5161                                                    $drive->{bps_max_length} || 1,
5162                                                    $drive->{bps_rd_max_length} || 1,
5163                                                    $drive->{bps_wr_max_length} || 1,
5164                                                    $drive->{iops_max_length} || 1,
5165                                                    $drive->{iops_rd_max_length} || 1,
5166                                                    $drive->{iops_wr_max_length} || 1);
5167
5168                     }
5169
5170                     return 1;
5171                 }
5172
5173             } else { # cdrom
5174
5175                 if ($drive->{file} eq 'none') {
5176                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5177                     if (drive_is_cloudinit($old_drive)) {
5178                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5179                     }
5180                 } else {
5181                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5182                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5183                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5184                 }
5185
5186                 return 1;
5187             }
5188         }
5189     }
5190
5191     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5192     # hotplug new disks
5193     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5194     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5195 }
5196
5197 sub vm_start {
5198     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5199         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5200
5201     PVE::QemuConfig->lock_config($vmid, sub {
5202         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5203
5204         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5205
5206         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5207
5208         PVE::QemuConfig->check_lock($conf)
5209             if !($skiplock || $is_suspended);
5210
5211         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5212
5213         # clean up leftover reboot request files
5214         eval { clear_reboot_request($vmid); };
5215         warn $@ if $@;
5216
5217         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5218             vmconfig_apply_pending($vmid, $conf, $storecfg);
5219             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5220         }
5221
5222         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5223
5224         my $defaults = load_defaults();
5225
5226         # set environment variable useful inside network script
5227         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5228
5229         my $local_volumes = {};
5230
5231         if ($targetstorage) {
5232             foreach_drive($conf, sub {
5233                 my ($ds, $drive) = @_;
5234
5235                 return if drive_is_cdrom($drive);
5236
5237                 my $volid = $drive->{file};
5238
5239                 return if !$volid;
5240
5241                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5242
5243                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5244                 return if $scfg->{shared};
5245                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5246             });
5247
5248             my $format = undef;
5249
5250             foreach my $opt (sort keys %$local_volumes) {
5251
5252                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5253                 my $drive = parse_drive($opt, $conf->{$opt});
5254
5255                 #if remote storage is specified, use default format
5256                 if ($targetstorage && $targetstorage ne "1") {
5257                     $storeid = $targetstorage;
5258                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5259                     $format = $defFormat;
5260                 } else {
5261                     #else we use same format than original
5262                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5263                     $format = qemu_img_format($scfg, $volid);
5264                 }
5265
5266                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5267                 my $newdrive = $drive;
5268                 $newdrive->{format} = $format;
5269                 $newdrive->{file} = $newvolid;
5270                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5271                 $local_volumes->{$opt} = $drivestr;
5272                 #pass drive to conf for command line
5273                 $conf->{$opt} = $drivestr;
5274             }
5275         }
5276
5277         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5278
5279         if ($is_suspended) {
5280             # enforce machine type on suspended vm to ensure HW compatibility
5281             $forcemachine = $conf->{runningmachine};
5282             print "Resuming suspended VM\n";
5283         }
5284
5285         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5286
5287         my $migration_ip;
5288         my $get_migration_ip = sub {
5289             my ($cidr, $nodename) = @_;
5290
5291             return $migration_ip if defined($migration_ip);
5292
5293             if (!defined($cidr)) {
5294                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5295                 $cidr = $dc_conf->{migration}->{network};
5296             }
5297
5298             if (defined($cidr)) {
5299                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5300
5301                 die "could not get IP: no address configured on local " .
5302                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5303
5304                 die "could not get IP: multiple addresses configured on local " .
5305                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5306
5307                 $migration_ip = @$ips[0];
5308             }
5309
5310             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5311                 if !defined($migration_ip);
5312
5313             return $migration_ip;
5314         };
5315
5316         my $migrate_uri;
5317         if ($statefile) {
5318             if ($statefile eq 'tcp') {
5319                 my $localip = "localhost";
5320                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5321                 my $nodename = PVE::INotify::nodename();
5322
5323                 if (!defined($migration_type)) {
5324                     if (defined($datacenterconf->{migration}->{type})) {
5325                         $migration_type = $datacenterconf->{migration}->{type};
5326                     } else {
5327                         $migration_type = 'secure';
5328                     }
5329                 }
5330
5331                 if ($migration_type eq 'insecure') {
5332                     $localip = $get_migration_ip->($migration_network, $nodename);
5333                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5334                 }
5335
5336                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5337                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5338                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5339                 push @$cmd, '-incoming', $migrate_uri;
5340                 push @$cmd, '-S';
5341
5342             } elsif ($statefile eq 'unix') {
5343                 # should be default for secure migrations as a ssh TCP forward
5344                 # tunnel is not deterministic reliable ready and fails regurarly
5345                 # to set up in time, so use UNIX socket forwards
5346                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5347                 unlink $socket_addr;
5348
5349                 $migrate_uri = "unix:$socket_addr";
5350
5351                 push @$cmd, '-incoming', $migrate_uri;
5352                 push @$cmd, '-S';
5353
5354             } elsif (-e $statefile) {
5355                 push @$cmd, '-loadstate', $statefile;
5356             } else {
5357                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5358                 push @$vollist, $statefile;
5359                 push @$cmd, '-loadstate', $statepath;
5360             }
5361         } elsif ($paused) {
5362             push @$cmd, '-S';
5363         }
5364
5365         # host pci devices
5366         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5367           my $d = parse_hostpci($conf->{"hostpci$i"});
5368           next if !$d;
5369           my $pcidevices = $d->{pciid};
5370           foreach my $pcidevice (@$pcidevices) {
5371                 my $pciid = $pcidevice->{id};
5372                 $pciid = "0000:$pciid" if $pciid !~ m/^[0-9a-f]{4}:/;
5373
5374                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5375                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5376                 die "no pci device info for device '$pciid'\n" if !$info;
5377
5378                 if ($d->{mdev}) {
5379                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5380                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5381                 } else {
5382                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5383                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5384                     die "can't reset pci device '$pciid'\n"
5385                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5386                 }
5387           }
5388         }
5389
5390         PVE::Storage::activate_volumes($storecfg, $vollist);
5391
5392         eval {
5393             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5394                 outfunc => sub {}, errfunc => sub {});
5395         };
5396         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5397         # timeout should be more than enough here...
5398         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5399
5400         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5401                                                   : $defaults->{cpuunits};
5402
5403         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5404         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5405
5406         my %properties = (
5407             Slice => 'qemu.slice',
5408             KillMode => 'none',
5409             CPUShares => $cpuunits
5410         );
5411
5412         if (my $cpulimit = $conf->{cpulimit}) {
5413             $properties{CPUQuota} = int($cpulimit * 100);
5414         }
5415         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5416
5417         my $run_qemu = sub {
5418             PVE::Tools::run_fork sub {
5419                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5420                 run_command($cmd, %run_params);
5421             };
5422         };
5423
5424         if ($conf->{hugepages}) {
5425
5426             my $code = sub {
5427                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5428                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5429
5430                 PVE::QemuServer::Memory::hugepages_mount();
5431                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5432
5433                 eval { $run_qemu->() };
5434                 if (my $err = $@) {
5435                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5436                     die $err;
5437                 }
5438
5439                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5440             };
5441             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5442
5443         } else {
5444             eval { $run_qemu->() };
5445         }
5446
5447         if (my $err = $@) {
5448             # deactivate volumes if start fails
5449             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5450             die "start failed: $err";
5451         }
5452
5453         print "migration listens on $migrate_uri\n" if $migrate_uri;
5454
5455         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5456             eval { mon_cmd($vmid, "cont"); };
5457             warn $@ if $@;
5458         }
5459
5460         #start nbd server for storage migration
5461         if ($targetstorage) {
5462             my $nodename = PVE::INotify::nodename();
5463             my $localip = $get_migration_ip->($migration_network, $nodename);
5464             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5465             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5466
5467             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5468
5469             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5470
5471             foreach my $opt (sort keys %$local_volumes) {
5472                 my $volid = $local_volumes->{$opt};
5473                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5474                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5475                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5476             }
5477         }
5478
5479         if ($migratedfrom) {
5480             eval {
5481                 set_migration_caps($vmid);
5482             };
5483             warn $@ if $@;
5484
5485             if ($spice_port) {
5486                 print "spice listens on port $spice_port\n";
5487                 if ($spice_ticket) {
5488                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5489                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5490                 }
5491             }
5492
5493         } else {
5494             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5495                 if !$statefile && $conf->{balloon};
5496
5497             foreach my $opt (keys %$conf) {
5498                 next if $opt !~  m/^net\d+$/;
5499                 my $nicconf = parse_net($conf->{$opt});
5500                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5501             }
5502         }
5503
5504         mon_cmd($vmid, 'qom-set',
5505                     path => "machine/peripheral/balloon0",
5506                     property => "guest-stats-polling-interval",
5507                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5508
5509         if ($is_suspended) {
5510             print "Resumed VM, removing state\n";
5511             if (my $vmstate = $conf->{vmstate}) {
5512                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5513                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5514             }
5515             delete $conf->@{qw(lock vmstate runningmachine)};
5516             PVE::QemuConfig->write_config($vmid, $conf);
5517         }
5518
5519         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5520     });
5521 }
5522
5523 sub vm_commandline {
5524     my ($storecfg, $vmid, $snapname) = @_;
5525
5526     my $conf = PVE::QemuConfig->load_config($vmid);
5527     my $forcemachine;
5528
5529     if ($snapname) {
5530         my $snapshot = $conf->{snapshots}->{$snapname};
5531         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5532
5533         # check for a 'runningmachine' in snapshot
5534         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5535
5536         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5537
5538         $conf = $snapshot;
5539     }
5540
5541     my $defaults = load_defaults();
5542
5543     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5544
5545     return PVE::Tools::cmd2string($cmd);
5546 }
5547
5548 sub vm_reset {
5549     my ($vmid, $skiplock) = @_;
5550
5551     PVE::QemuConfig->lock_config($vmid, sub {
5552
5553         my $conf = PVE::QemuConfig->load_config($vmid);
5554
5555         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5556
5557         mon_cmd($vmid, "system_reset");
5558     });
5559 }
5560
5561 sub get_vm_volumes {
5562     my ($conf) = @_;
5563
5564     my $vollist = [];
5565     foreach_volid($conf, sub {
5566         my ($volid, $attr) = @_;
5567
5568         return if $volid =~ m|^/|;
5569
5570         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5571         return if !$sid;
5572
5573         push @$vollist, $volid;
5574     });
5575
5576     return $vollist;
5577 }
5578
5579 sub vm_stop_cleanup {
5580     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5581
5582     eval {
5583
5584         if (!$keepActive) {
5585             my $vollist = get_vm_volumes($conf);
5586             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5587         }
5588
5589         foreach my $ext (qw(mon qmp pid vnc qga)) {
5590             unlink "/var/run/qemu-server/${vmid}.$ext";
5591         }
5592
5593         if ($conf->{ivshmem}) {
5594             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5595             # just delete it for now, VMs which have this already open do not
5596             # are affected, but new VMs will get a separated one. If this
5597             # becomes an issue we either add some sort of ref-counting or just
5598             # add a "don't delete on stop" flag to the ivshmem format.
5599             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5600         }
5601
5602         foreach my $key (keys %$conf) {
5603             next if $key !~ m/^hostpci(\d+)$/;
5604             my $hostpciindex = $1;
5605             my $d = parse_hostpci($conf->{$key});
5606             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5607
5608             foreach my $pci (@{$d->{pciid}}) {
5609                 my $pciid = $pci->{id};
5610                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5611             }
5612         }
5613
5614         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5615     };
5616     warn $@ if $@; # avoid errors - just warn
5617 }
5618
5619 # call only in locked context
5620 sub _do_vm_stop {
5621     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5622
5623     my $pid = check_running($vmid, $nocheck);
5624     return if !$pid;
5625
5626     my $conf;
5627     if (!$nocheck) {
5628         $conf = PVE::QemuConfig->load_config($vmid);
5629         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5630         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5631             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5632             $timeout = $opts->{down} if $opts->{down};
5633         }
5634         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5635     }
5636
5637     eval {
5638         if ($shutdown) {
5639             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5640                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5641             } else {
5642                 mon_cmd($vmid, "system_powerdown");
5643             }
5644         } else {
5645             mon_cmd($vmid, "quit");
5646         }
5647     };
5648     my $err = $@;
5649
5650     if (!$err) {
5651         $timeout = 60 if !defined($timeout);
5652
5653         my $count = 0;
5654         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5655             $count++;
5656             sleep 1;
5657         }
5658
5659         if ($count >= $timeout) {
5660             if ($force) {
5661                 warn "VM still running - terminating now with SIGTERM\n";
5662                 kill 15, $pid;
5663             } else {
5664                 die "VM quit/powerdown failed - got timeout\n";
5665             }
5666         } else {
5667             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5668             return;
5669         }
5670     } else {
5671         if ($force) {
5672             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5673             kill 15, $pid;
5674         } else {
5675             die "VM quit/powerdown failed\n";
5676         }
5677     }
5678
5679     # wait again
5680     $timeout = 10;
5681
5682     my $count = 0;
5683     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5684         $count++;
5685         sleep 1;
5686     }
5687
5688     if ($count >= $timeout) {
5689         warn "VM still running - terminating now with SIGKILL\n";
5690         kill 9, $pid;
5691         sleep 1;
5692     }
5693
5694     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5695 }
5696
5697 # Note: use $nocheck to skip tests if VM configuration file exists.
5698 # We need that when migration VMs to other nodes (files already moved)
5699 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5700 sub vm_stop {
5701     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5702
5703     $force = 1 if !defined($force) && !$shutdown;
5704
5705     if ($migratedfrom){
5706         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5707         kill 15, $pid if $pid;
5708         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5709         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5710         return;
5711     }
5712
5713     PVE::QemuConfig->lock_config($vmid, sub {
5714         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5715    });
5716 }
5717
5718 sub vm_reboot {
5719     my ($vmid, $timeout) = @_;
5720
5721     PVE::QemuConfig->lock_config($vmid, sub {
5722         eval {
5723
5724             # only reboot if running, as qmeventd starts it again on a stop event
5725             return if !check_running($vmid);
5726
5727             create_reboot_request($vmid);
5728
5729             my $storecfg = PVE::Storage::config();
5730             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5731
5732         };
5733         if (my $err = $@) {
5734             # avoid that the next normal shutdown will be confused for a reboot
5735             clear_reboot_request($vmid);
5736             die $err;
5737         }
5738    });
5739 }
5740
5741 sub vm_suspend {
5742     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5743
5744     my $conf;
5745     my $path;
5746     my $storecfg;
5747     my $vmstate;
5748
5749     PVE::QemuConfig->lock_config($vmid, sub {
5750
5751         $conf = PVE::QemuConfig->load_config($vmid);
5752
5753         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5754         PVE::QemuConfig->check_lock($conf)
5755             if !($skiplock || $is_backing_up);
5756
5757         die "cannot suspend to disk during backup\n"
5758             if $is_backing_up && $includestate;
5759
5760         if ($includestate) {
5761             $conf->{lock} = 'suspending';
5762             my $date = strftime("%Y-%m-%d", localtime(time()));
5763             $storecfg = PVE::Storage::config();
5764             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5765             $path = PVE::Storage::path($storecfg, $vmstate);
5766             PVE::QemuConfig->write_config($vmid, $conf);
5767         } else {
5768             mon_cmd($vmid, "stop");
5769         }
5770     });
5771
5772     if ($includestate) {
5773         # save vm state
5774         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5775
5776         eval {
5777             mon_cmd($vmid, "savevm-start", statefile => $path);
5778             for(;;) {
5779                 my $state = mon_cmd($vmid, "query-savevm");
5780                 if (!$state->{status}) {
5781                     die "savevm not active\n";
5782                 } elsif ($state->{status} eq 'active') {
5783                     sleep(1);
5784                     next;
5785                 } elsif ($state->{status} eq 'completed') {
5786                     print "State saved, quitting\n";
5787                     last;
5788                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5789                     die "query-savevm failed with error '$state->{error}'\n"
5790                 } else {
5791                     die "query-savevm returned status '$state->{status}'\n";
5792                 }
5793             }
5794         };
5795         my $err = $@;
5796
5797         PVE::QemuConfig->lock_config($vmid, sub {
5798             $conf = PVE::QemuConfig->load_config($vmid);
5799             if ($err) {
5800                 # cleanup, but leave suspending lock, to indicate something went wrong
5801                 eval {
5802                     mon_cmd($vmid, "savevm-end");
5803                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5804                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5805                     delete $conf->@{qw(vmstate runningmachine)};
5806                     PVE::QemuConfig->write_config($vmid, $conf);
5807                 };
5808                 warn $@ if $@;
5809                 die $err;
5810             }
5811
5812             die "lock changed unexpectedly\n"
5813                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5814
5815             mon_cmd($vmid, "quit");
5816             $conf->{lock} = 'suspended';
5817             PVE::QemuConfig->write_config($vmid, $conf);
5818         });
5819     }
5820 }
5821
5822 sub vm_resume {
5823     my ($vmid, $skiplock, $nocheck) = @_;
5824
5825     PVE::QemuConfig->lock_config($vmid, sub {
5826         my $res = mon_cmd($vmid, 'query-status');
5827         my $resume_cmd = 'cont';
5828
5829         if ($res->{status} && $res->{status} eq 'suspended') {
5830             $resume_cmd = 'system_wakeup';
5831         }
5832
5833         if (!$nocheck) {
5834
5835             my $conf = PVE::QemuConfig->load_config($vmid);
5836
5837             PVE::QemuConfig->check_lock($conf)
5838                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5839         }
5840
5841         mon_cmd($vmid, $resume_cmd);
5842     });
5843 }
5844
5845 sub vm_sendkey {
5846     my ($vmid, $skiplock, $key) = @_;
5847
5848     PVE::QemuConfig->lock_config($vmid, sub {
5849
5850         my $conf = PVE::QemuConfig->load_config($vmid);
5851
5852         # there is no qmp command, so we use the human monitor command
5853         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5854         die $res if $res ne '';
5855     });
5856 }
5857
5858 # vzdump restore implementaion
5859
5860 sub tar_archive_read_firstfile {
5861     my $archive = shift;
5862
5863     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5864
5865     # try to detect archive type first
5866     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5867         die "unable to open file '$archive'\n";
5868     my $firstfile = <$fh>;
5869     kill 15, $pid;
5870     close $fh;
5871
5872     die "ERROR: archive contaions no data\n" if !$firstfile;
5873     chomp $firstfile;
5874
5875     return $firstfile;
5876 }
5877
5878 sub tar_restore_cleanup {
5879     my ($storecfg, $statfile) = @_;
5880
5881     print STDERR "starting cleanup\n";
5882
5883     if (my $fd = IO::File->new($statfile, "r")) {
5884         while (defined(my $line = <$fd>)) {
5885             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5886                 my $volid = $2;
5887                 eval {
5888                     if ($volid =~ m|^/|) {
5889                         unlink $volid || die 'unlink failed\n';
5890                     } else {
5891                         PVE::Storage::vdisk_free($storecfg, $volid);
5892                     }
5893                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5894                 };
5895                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5896             } else {
5897                 print STDERR "unable to parse line in statfile - $line";
5898             }
5899         }
5900         $fd->close();
5901     }
5902 }
5903
5904 sub restore_archive {
5905     my ($archive, $vmid, $user, $opts) = @_;
5906
5907     my $format = $opts->{format};
5908     my $comp;
5909
5910     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5911         $format = 'tar' if !$format;
5912         $comp = 'gzip';
5913     } elsif ($archive =~ m/\.tar$/) {
5914         $format = 'tar' if !$format;
5915     } elsif ($archive =~ m/.tar.lzo$/) {
5916         $format = 'tar' if !$format;
5917         $comp = 'lzop';
5918     } elsif ($archive =~ m/\.vma$/) {
5919         $format = 'vma' if !$format;
5920     } elsif ($archive =~ m/\.vma\.gz$/) {
5921         $format = 'vma' if !$format;
5922         $comp = 'gzip';
5923     } elsif ($archive =~ m/\.vma\.lzo$/) {
5924         $format = 'vma' if !$format;
5925         $comp = 'lzop';
5926     } else {
5927         $format = 'vma' if !$format; # default
5928     }
5929
5930     # try to detect archive format
5931     if ($format eq 'tar') {
5932         return restore_tar_archive($archive, $vmid, $user, $opts);
5933     } else {
5934         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5935     }
5936 }
5937
5938 sub restore_update_config_line {
5939     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5940
5941     return if $line =~ m/^\#qmdump\#/;
5942     return if $line =~ m/^\#vzdump\#/;
5943     return if $line =~ m/^lock:/;
5944     return if $line =~ m/^unused\d+:/;
5945     return if $line =~ m/^parent:/;
5946
5947     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5948     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5949         # try to convert old 1.X settings
5950         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5951         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5952             my ($model, $macaddr) = split(/\=/, $devconfig);
5953             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5954             my $net = {
5955                 model => $model,
5956                 bridge => "vmbr$ind",
5957                 macaddr => $macaddr,
5958             };
5959             my $netstr = print_net($net);
5960
5961             print $outfd "net$cookie->{netcount}: $netstr\n";
5962             $cookie->{netcount}++;
5963         }
5964     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5965         my ($id, $netstr) = ($1, $2);
5966         my $net = parse_net($netstr);
5967         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5968         $netstr = print_net($net);
5969         print $outfd "$id: $netstr\n";
5970     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5971         my $virtdev = $1;
5972         my $value = $3;
5973         my $di = parse_drive($virtdev, $value);
5974         if (defined($di->{backup}) && !$di->{backup}) {
5975             print $outfd "#$line";
5976         } elsif ($map->{$virtdev}) {
5977             delete $di->{format}; # format can change on restore
5978             $di->{file} = $map->{$virtdev};
5979             $value = print_drive($vmid, $di);
5980             print $outfd "$virtdev: $value\n";
5981         } else {
5982             print $outfd $line;
5983         }
5984     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5985         my $vmgenid = $1;
5986         if ($vmgenid ne '0') {
5987             # always generate a new vmgenid if there was a valid one setup
5988             $vmgenid = generate_uuid();
5989         }
5990         print $outfd "vmgenid: $vmgenid\n";
5991     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5992         my ($uuid, $uuid_str);
5993         UUID::generate($uuid);
5994         UUID::unparse($uuid, $uuid_str);
5995         my $smbios1 = parse_smbios1($2);
5996         $smbios1->{uuid} = $uuid_str;
5997         print $outfd $1.print_smbios1($smbios1)."\n";
5998     } else {
5999         print $outfd $line;
6000     }
6001 }
6002
6003 sub scan_volids {
6004     my ($cfg, $vmid) = @_;
6005
6006     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6007
6008     my $volid_hash = {};
6009     foreach my $storeid (keys %$info) {
6010         foreach my $item (@{$info->{$storeid}}) {
6011             next if !($item->{volid} && $item->{size});
6012             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6013             $volid_hash->{$item->{volid}} = $item;
6014         }
6015     }
6016
6017     return $volid_hash;
6018 }
6019
6020 sub is_volume_in_use {
6021     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6022
6023     my $path = PVE::Storage::path($storecfg, $volid);
6024
6025     my $scan_config = sub {
6026         my ($cref, $snapname) = @_;
6027
6028         foreach my $key (keys %$cref) {
6029             my $value = $cref->{$key};
6030             if (is_valid_drivename($key)) {
6031                 next if $skip_drive && $key eq $skip_drive;
6032                 my $drive = parse_drive($key, $value);
6033                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6034                 return 1 if $volid eq $drive->{file};
6035                 if ($drive->{file} =~ m!^/!) {
6036                     return 1 if $drive->{file} eq $path;
6037                 } else {
6038                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6039                     next if !$storeid;
6040                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6041                     next if !$scfg;
6042                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6043                 }
6044             }
6045         }
6046
6047         return 0;
6048     };
6049
6050     return 1 if &$scan_config($conf);
6051
6052     undef $skip_drive;
6053
6054     foreach my $snapname (keys %{$conf->{snapshots}}) {
6055         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6056     }
6057
6058     return 0;
6059 }
6060
6061 sub update_disksize {
6062     my ($vmid, $conf, $volid_hash) = @_;
6063
6064     my $changes;
6065     my $prefix = "VM $vmid:";
6066
6067     # used and unused disks
6068     my $referenced = {};
6069
6070     # Note: it is allowed to define multiple storages with same path (alias), so
6071     # we need to check both 'volid' and real 'path' (two different volid can point
6072     # to the same path).
6073
6074     my $referencedpath = {};
6075
6076     # update size info
6077     foreach my $opt (keys %$conf) {
6078         if (is_valid_drivename($opt)) {
6079             my $drive = parse_drive($opt, $conf->{$opt});
6080             my $volid = $drive->{file};
6081             next if !$volid;
6082
6083             $referenced->{$volid} = 1;
6084             if ($volid_hash->{$volid} &&
6085                 (my $path = $volid_hash->{$volid}->{path})) {
6086                 $referencedpath->{$path} = 1;
6087             }
6088
6089             next if drive_is_cdrom($drive);
6090             next if !$volid_hash->{$volid};
6091
6092             $drive->{size} = $volid_hash->{$volid}->{size};
6093             my $new = print_drive($vmid, $drive);
6094             if ($new ne $conf->{$opt}) {
6095                 $changes = 1;
6096                 $conf->{$opt} = $new;
6097                 print "$prefix update disk '$opt' information.\n";
6098             }
6099         }
6100     }
6101
6102     # remove 'unusedX' entry if volume is used
6103     foreach my $opt (keys %$conf) {
6104         next if $opt !~ m/^unused\d+$/;
6105         my $volid = $conf->{$opt};
6106         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6107         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6108             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6109             $changes = 1;
6110             delete $conf->{$opt};
6111         }
6112
6113         $referenced->{$volid} = 1;
6114         $referencedpath->{$path} = 1 if $path;
6115     }
6116
6117     foreach my $volid (sort keys %$volid_hash) {
6118         next if $volid =~ m/vm-$vmid-state-/;
6119         next if $referenced->{$volid};
6120         my $path = $volid_hash->{$volid}->{path};
6121         next if !$path; # just to be sure
6122         next if $referencedpath->{$path};
6123         $changes = 1;
6124         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6125         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6126         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6127     }
6128
6129     return $changes;
6130 }
6131
6132 sub rescan {
6133     my ($vmid, $nolock, $dryrun) = @_;
6134
6135     my $cfg = PVE::Storage::config();
6136
6137     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6138     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6139     foreach my $stor (keys %{$cfg->{ids}}) {
6140         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6141     }
6142
6143     print "rescan volumes...\n";
6144     my $volid_hash = scan_volids($cfg, $vmid);
6145
6146     my $updatefn =  sub {
6147         my ($vmid) = @_;
6148
6149         my $conf = PVE::QemuConfig->load_config($vmid);
6150
6151         PVE::QemuConfig->check_lock($conf);
6152
6153         my $vm_volids = {};
6154         foreach my $volid (keys %$volid_hash) {
6155             my $info = $volid_hash->{$volid};
6156             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6157         }
6158
6159         my $changes = update_disksize($vmid, $conf, $vm_volids);
6160
6161         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6162     };
6163
6164     if (defined($vmid)) {
6165         if ($nolock) {
6166             &$updatefn($vmid);
6167         } else {
6168             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6169         }
6170     } else {
6171         my $vmlist = config_list();
6172         foreach my $vmid (keys %$vmlist) {
6173             if ($nolock) {
6174                 &$updatefn($vmid);
6175             } else {
6176                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6177             }
6178         }
6179     }
6180 }
6181
6182 sub restore_vma_archive {
6183     my ($archive, $vmid, $user, $opts, $comp) = @_;
6184
6185     my $readfrom = $archive;
6186
6187     my $cfg = PVE::Storage::config();
6188     my $commands = [];
6189     my $bwlimit = $opts->{bwlimit};
6190
6191     my $dbg_cmdstring = '';
6192     my $add_pipe = sub {
6193         my ($cmd) = @_;
6194         push @$commands, $cmd;
6195         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6196         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6197         $readfrom = '-';
6198     };
6199
6200     my $input = undef;
6201     if ($archive eq '-') {
6202         $input = '<&STDIN';
6203     } else {
6204         # If we use a backup from a PVE defined storage we also consider that
6205         # storage's rate limit:
6206         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6207         if (defined($volid)) {
6208             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6209             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6210             if ($readlimit) {
6211                 print STDERR "applying read rate limit: $readlimit\n";
6212                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6213                 $add_pipe->($cstream);
6214             }
6215         }
6216     }
6217
6218     if ($comp) {
6219         my $cmd;
6220         if ($comp eq 'gzip') {
6221             $cmd = ['zcat', $readfrom];
6222         } elsif ($comp eq 'lzop') {
6223             $cmd = ['lzop', '-d', '-c', $readfrom];
6224         } else {
6225             die "unknown compression method '$comp'\n";
6226         }
6227         $add_pipe->($cmd);
6228     }
6229
6230     my $tmpdir = "/var/tmp/vzdumptmp$$";
6231     rmtree $tmpdir;
6232
6233     # disable interrupts (always do cleanups)
6234     local $SIG{INT} =
6235         local $SIG{TERM} =
6236         local $SIG{QUIT} =
6237         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6238
6239     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6240     POSIX::mkfifo($mapfifo, 0600);
6241     my $fifofh;
6242
6243     my $openfifo = sub {
6244         open($fifofh, '>', $mapfifo) || die $!;
6245     };
6246
6247     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6248
6249     my $oldtimeout;
6250     my $timeout = 5;
6251
6252     my $devinfo = {};
6253
6254     my $rpcenv = PVE::RPCEnvironment::get();
6255
6256     my $conffile = PVE::QemuConfig->config_file($vmid);
6257     my $tmpfn = "$conffile.$$.tmp";
6258
6259     # Note: $oldconf is undef if VM does not exists
6260     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6261     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6262
6263     my %storage_limits;
6264
6265     my $print_devmap = sub {
6266         my $virtdev_hash = {};
6267
6268         my $cfgfn = "$tmpdir/qemu-server.conf";
6269
6270         # we can read the config - that is already extracted
6271         my $fh = IO::File->new($cfgfn, "r") ||
6272             "unable to read qemu-server.conf - $!\n";
6273
6274         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6275         if (-f $fwcfgfn) {
6276             my $pve_firewall_dir = '/etc/pve/firewall';
6277             mkdir $pve_firewall_dir; # make sure the dir exists
6278             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6279         }
6280
6281         while (defined(my $line = <$fh>)) {
6282             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6283                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6284                 die "archive does not contain data for drive '$virtdev'\n"
6285                     if !$devinfo->{$devname};
6286                 if (defined($opts->{storage})) {
6287                     $storeid = $opts->{storage} || 'local';
6288                 } elsif (!$storeid) {
6289                     $storeid = 'local';
6290                 }
6291                 $format = 'raw' if !$format;
6292                 $devinfo->{$devname}->{devname} = $devname;
6293                 $devinfo->{$devname}->{virtdev} = $virtdev;
6294                 $devinfo->{$devname}->{format} = $format;
6295                 $devinfo->{$devname}->{storeid} = $storeid;
6296
6297                 # check permission on storage
6298                 my $pool = $opts->{pool}; # todo: do we need that?
6299                 if ($user ne 'root@pam') {
6300                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6301                 }
6302
6303                 $storage_limits{$storeid} = $bwlimit;
6304
6305                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6306             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6307                 my $virtdev = $1;
6308                 my $drive = parse_drive($virtdev, $2);
6309                 if (drive_is_cloudinit($drive)) {
6310                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6311                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6312                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6313
6314                     my $d = {
6315                         format => $format,
6316                         storeid => $opts->{storage} // $storeid,
6317                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6318                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6319                         name => "vm-$vmid-cloudinit",
6320                         is_cloudinit => 1,
6321                     };
6322                     $virtdev_hash->{$virtdev} = $d;
6323                 }
6324             }
6325         }
6326
6327         foreach my $key (keys %storage_limits) {
6328             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6329             next if !$limit;
6330             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6331             $storage_limits{$key} = $limit * 1024;
6332         }
6333
6334         foreach my $devname (keys %$devinfo) {
6335             die "found no device mapping information for device '$devname'\n"
6336                 if !$devinfo->{$devname}->{virtdev};
6337         }
6338
6339         # create empty/temp config
6340         if ($oldconf) {
6341             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6342             foreach_drive($oldconf, sub {
6343                 my ($ds, $drive) = @_;
6344
6345                 return if drive_is_cdrom($drive, 1);
6346
6347                 my $volid = $drive->{file};
6348                 return if !$volid || $volid =~ m|^/|;
6349
6350                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6351                 return if !$path || !$owner || ($owner != $vmid);
6352
6353                 # Note: only delete disk we want to restore
6354                 # other volumes will become unused
6355                 if ($virtdev_hash->{$ds}) {
6356                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6357                     if (my $err = $@) {
6358                         warn $err;
6359                     }
6360                 }
6361             });
6362
6363             # delete vmstate files, after the restore we have no snapshots anymore
6364             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6365                 my $snap = $oldconf->{snapshots}->{$snapname};
6366                 if ($snap->{vmstate}) {
6367                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6368                     if (my $err = $@) {
6369                         warn $err;
6370                     }
6371                 }
6372             }
6373         }
6374
6375         my $map = {};
6376         foreach my $virtdev (sort keys %$virtdev_hash) {
6377             my $d = $virtdev_hash->{$virtdev};
6378             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6379             my $storeid = $d->{storeid};
6380             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6381
6382             my $map_opts = '';
6383             if (my $limit = $storage_limits{$storeid}) {
6384                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6385             }
6386
6387             # test if requested format is supported
6388             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6389             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6390             $d->{format} = $defFormat if !$supported;
6391
6392             my $name;
6393             if ($d->{is_cloudinit}) {
6394                 $name = $d->{name};
6395                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6396             }
6397
6398             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6399             print STDERR "new volume ID is '$volid'\n";
6400             $d->{volid} = $volid;
6401
6402             PVE::Storage::activate_volumes($cfg, [$volid]);
6403
6404             my $write_zeros = 1;
6405             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6406                 $write_zeros = 0;
6407             }
6408
6409             if (!$d->{is_cloudinit}) {
6410                 my $path = PVE::Storage::path($cfg, $volid);
6411
6412                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6413
6414                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6415             }
6416             $map->{$virtdev} = $volid;
6417         }
6418
6419         $fh->seek(0, 0) || die "seek failed - $!\n";
6420
6421         my $outfd = new IO::File ($tmpfn, "w") ||
6422             die "unable to write config for VM $vmid\n";
6423
6424         my $cookie = { netcount => 0 };
6425         while (defined(my $line = <$fh>)) {
6426             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6427         }
6428
6429         $fh->close();
6430         $outfd->close();
6431     };
6432
6433     eval {
6434         # enable interrupts
6435         local $SIG{INT} =
6436             local $SIG{TERM} =
6437             local $SIG{QUIT} =
6438             local $SIG{HUP} =
6439             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6440         local $SIG{ALRM} = sub { die "got timeout\n"; };
6441
6442         $oldtimeout = alarm($timeout);
6443
6444         my $parser = sub {
6445             my $line = shift;
6446
6447             print "$line\n";
6448
6449             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6450                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6451                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6452             } elsif ($line =~ m/^CTIME: /) {
6453                 # we correctly received the vma config, so we can disable
6454                 # the timeout now for disk allocation (set to 10 minutes, so
6455                 # that we always timeout if something goes wrong)
6456                 alarm(600);
6457                 &$print_devmap();
6458                 print $fifofh "done\n";
6459                 my $tmp = $oldtimeout || 0;
6460                 $oldtimeout = undef;
6461                 alarm($tmp);
6462                 close($fifofh);
6463             }
6464         };
6465
6466         print "restore vma archive: $dbg_cmdstring\n";
6467         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6468     };
6469     my $err = $@;
6470
6471     alarm($oldtimeout) if $oldtimeout;
6472
6473     my $vollist = [];
6474     foreach my $devname (keys %$devinfo) {
6475         my $volid = $devinfo->{$devname}->{volid};
6476         push @$vollist, $volid if $volid;
6477     }
6478
6479     PVE::Storage::deactivate_volumes($cfg, $vollist);
6480
6481     unlink $mapfifo;
6482
6483     if ($err) {
6484         rmtree $tmpdir;
6485         unlink $tmpfn;
6486
6487         foreach my $devname (keys %$devinfo) {
6488             my $volid = $devinfo->{$devname}->{volid};
6489             next if !$volid;
6490             eval {
6491                 if ($volid =~ m|^/|) {
6492                     unlink $volid || die 'unlink failed\n';
6493                 } else {
6494                     PVE::Storage::vdisk_free($cfg, $volid);
6495                 }
6496                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6497             };
6498             print STDERR "unable to cleanup '$volid' - $@" if $@;
6499         }
6500         die $err;
6501     }
6502
6503     rmtree $tmpdir;
6504
6505     rename($tmpfn, $conffile) ||
6506         die "unable to commit configuration file '$conffile'\n";
6507
6508     PVE::Cluster::cfs_update(); # make sure we read new file
6509
6510     eval { rescan($vmid, 1); };
6511     warn $@ if $@;
6512 }
6513
6514 sub restore_tar_archive {
6515     my ($archive, $vmid, $user, $opts) = @_;
6516
6517     if ($archive ne '-') {
6518         my $firstfile = tar_archive_read_firstfile($archive);
6519         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6520             if $firstfile ne 'qemu-server.conf';
6521     }
6522
6523     my $storecfg = PVE::Storage::config();
6524
6525     # avoid zombie disks when restoring over an existing VM -> cleanup first
6526     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6527     # skiplock=1 because qmrestore has set the 'create' lock itself already
6528     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6529     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6530
6531     my $tocmd = "/usr/lib/qemu-server/qmextract";
6532
6533     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6534     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6535     $tocmd .= ' --prealloc' if $opts->{prealloc};
6536     $tocmd .= ' --info' if $opts->{info};
6537
6538     # tar option "xf" does not autodetect compression when read from STDIN,
6539     # so we pipe to zcat
6540     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6541         PVE::Tools::shellquote("--to-command=$tocmd");
6542
6543     my $tmpdir = "/var/tmp/vzdumptmp$$";
6544     mkpath $tmpdir;
6545
6546     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6547     local $ENV{VZDUMP_VMID} = $vmid;
6548     local $ENV{VZDUMP_USER} = $user;
6549
6550     my $conffile = PVE::QemuConfig->config_file($vmid);
6551     my $tmpfn = "$conffile.$$.tmp";
6552
6553     # disable interrupts (always do cleanups)
6554     local $SIG{INT} =
6555         local $SIG{TERM} =
6556         local $SIG{QUIT} =
6557         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6558
6559     eval {
6560         # enable interrupts
6561         local $SIG{INT} =
6562             local $SIG{TERM} =
6563             local $SIG{QUIT} =
6564             local $SIG{HUP} =
6565             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6566
6567         if ($archive eq '-') {
6568             print "extracting archive from STDIN\n";
6569             run_command($cmd, input => "<&STDIN");
6570         } else {
6571             print "extracting archive '$archive'\n";
6572             run_command($cmd);
6573         }
6574
6575         return if $opts->{info};
6576
6577         # read new mapping
6578         my $map = {};
6579         my $statfile = "$tmpdir/qmrestore.stat";
6580         if (my $fd = IO::File->new($statfile, "r")) {
6581             while (defined (my $line = <$fd>)) {
6582                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6583                     $map->{$1} = $2 if $1;
6584                 } else {
6585                     print STDERR "unable to parse line in statfile - $line\n";
6586                 }
6587             }
6588             $fd->close();
6589         }
6590
6591         my $confsrc = "$tmpdir/qemu-server.conf";
6592
6593         my $srcfd = new IO::File($confsrc, "r") ||
6594             die "unable to open file '$confsrc'\n";
6595
6596         my $outfd = new IO::File ($tmpfn, "w") ||
6597             die "unable to write config for VM $vmid\n";
6598
6599         my $cookie = { netcount => 0 };
6600         while (defined (my $line = <$srcfd>)) {
6601             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6602         }
6603
6604         $srcfd->close();
6605         $outfd->close();
6606     };
6607     if (my $err = $@) {
6608         unlink $tmpfn;
6609         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6610         die $err;
6611     }
6612
6613     rmtree $tmpdir;
6614
6615     rename $tmpfn, $conffile ||
6616         die "unable to commit configuration file '$conffile'\n";
6617
6618     PVE::Cluster::cfs_update(); # make sure we read new file
6619
6620     eval { rescan($vmid, 1); };
6621     warn $@ if $@;
6622 };
6623
6624 sub foreach_storage_used_by_vm {
6625     my ($conf, $func) = @_;
6626
6627     my $sidhash = {};
6628
6629     foreach_drive($conf, sub {
6630         my ($ds, $drive) = @_;
6631         return if drive_is_cdrom($drive);
6632
6633         my $volid = $drive->{file};
6634
6635         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6636         $sidhash->{$sid} = $sid if $sid;
6637     });
6638
6639     foreach my $sid (sort keys %$sidhash) {
6640         &$func($sid);
6641     }
6642 }
6643
6644 my $qemu_snap_storage = {
6645     rbd => 1,
6646 };
6647 sub do_snapshots_with_qemu {
6648     my ($storecfg, $volid) = @_;
6649
6650     my $storage_name = PVE::Storage::parse_volume_id($volid);
6651     my $scfg = $storecfg->{ids}->{$storage_name};
6652
6653     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6654         return 1;
6655     }
6656
6657     if ($volid =~ m/\.(qcow2|qed)$/){
6658         return 1;
6659     }
6660
6661     return undef;
6662 }
6663
6664 sub qga_check_running {
6665     my ($vmid, $nowarn) = @_;
6666
6667     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6668     if ($@) {
6669         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6670         return 0;
6671     }
6672     return 1;
6673 }
6674
6675 sub template_create {
6676     my ($vmid, $conf, $disk) = @_;
6677
6678     my $storecfg = PVE::Storage::config();
6679
6680     foreach_drive($conf, sub {
6681         my ($ds, $drive) = @_;
6682
6683         return if drive_is_cdrom($drive);
6684         return if $disk && $ds ne $disk;
6685
6686         my $volid = $drive->{file};
6687         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6688
6689         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6690         $drive->{file} = $voliddst;
6691         $conf->{$ds} = print_drive($vmid, $drive);
6692         PVE::QemuConfig->write_config($vmid, $conf);
6693     });
6694 }
6695
6696 sub convert_iscsi_path {
6697     my ($path) = @_;
6698
6699     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6700         my $portal = $1;
6701         my $target = $2;
6702         my $lun = $3;
6703
6704         my $initiator_name = get_initiator_name();
6705
6706         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6707                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6708     }
6709
6710     die "cannot convert iscsi path '$path', unkown format\n";
6711 }
6712
6713 sub qemu_img_convert {
6714     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6715
6716     my $storecfg = PVE::Storage::config();
6717     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6718     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6719
6720     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6721
6722     my $cachemode;
6723     my $src_path;
6724     my $src_is_iscsi = 0;
6725     my $src_format;
6726
6727     if ($src_storeid) {
6728         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6729         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6730         $src_format = qemu_img_format($src_scfg, $src_volname);
6731         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6732         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6733         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6734     } elsif (-f $src_volid) {
6735         $src_path = $src_volid;
6736         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6737             $src_format = $1;
6738         }
6739     }
6740
6741     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6742
6743     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6744     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6745     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6746     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6747
6748     my $cmd = [];
6749     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6750     push @$cmd, '-l', "snapshot.name=$snapname"
6751         if $snapname && $src_format && $src_format eq "qcow2";
6752     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6753     push @$cmd, '-T', $cachemode if defined($cachemode);
6754
6755     if ($src_is_iscsi) {
6756         push @$cmd, '--image-opts';
6757         $src_path = convert_iscsi_path($src_path);
6758     } elsif ($src_format) {
6759         push @$cmd, '-f', $src_format;
6760     }
6761
6762     if ($dst_is_iscsi) {
6763         push @$cmd, '--target-image-opts';
6764         $dst_path = convert_iscsi_path($dst_path);
6765     } else {
6766         push @$cmd, '-O', $dst_format;
6767     }
6768
6769     push @$cmd, $src_path;
6770
6771     if (!$dst_is_iscsi && $is_zero_initialized) {
6772         push @$cmd, "zeroinit:$dst_path";
6773     } else {
6774         push @$cmd, $dst_path;
6775     }
6776
6777     my $parser = sub {
6778         my $line = shift;
6779         if($line =~ m/\((\S+)\/100\%\)/){
6780             my $percent = $1;
6781             my $transferred = int($size * $percent / 100);
6782             my $remaining = $size - $transferred;
6783
6784             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6785         }
6786
6787     };
6788
6789     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6790     my $err = $@;
6791     die "copy failed: $err" if $err;
6792 }
6793
6794 sub qemu_img_format {
6795     my ($scfg, $volname) = @_;
6796
6797     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6798         return $1;
6799     } else {
6800         return "raw";
6801     }
6802 }
6803
6804 sub qemu_drive_mirror {
6805     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6806
6807     $jobs = {} if !$jobs;
6808
6809     my $qemu_target;
6810     my $format;
6811     $jobs->{"drive-$drive"} = {};
6812
6813     if ($dst_volid =~ /^nbd:/) {
6814         $qemu_target = $dst_volid;
6815         $format = "nbd";
6816     } else {
6817         my $storecfg = PVE::Storage::config();
6818         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6819
6820         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6821
6822         $format = qemu_img_format($dst_scfg, $dst_volname);
6823
6824         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6825
6826         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6827     }
6828
6829     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6830     $opts->{format} = $format if $format;
6831
6832     if (defined($bwlimit)) {
6833         $opts->{speed} = $bwlimit * 1024;
6834         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6835     } else {
6836         print "drive mirror is starting for drive-$drive\n";
6837     }
6838
6839     # if a job already runs for this device we get an error, catch it for cleanup
6840     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6841     if (my $err = $@) {
6842         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6843         warn "$@\n" if $@;
6844         die "mirroring error: $err\n";
6845     }
6846
6847     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6848 }
6849
6850 sub qemu_drive_mirror_monitor {
6851     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6852
6853     eval {
6854         my $err_complete = 0;
6855
6856         while (1) {
6857             die "storage migration timed out\n" if $err_complete > 300;
6858
6859             my $stats = mon_cmd($vmid, "query-block-jobs");
6860
6861             my $running_mirror_jobs = {};
6862             foreach my $stat (@$stats) {
6863                 next if $stat->{type} ne 'mirror';
6864                 $running_mirror_jobs->{$stat->{device}} = $stat;
6865             }
6866
6867             my $readycounter = 0;
6868
6869             foreach my $job (keys %$jobs) {
6870
6871                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6872                     print "$job : finished\n";
6873                     delete $jobs->{$job};
6874                     next;
6875                 }
6876
6877                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6878
6879                 my $busy = $running_mirror_jobs->{$job}->{busy};
6880                 my $ready = $running_mirror_jobs->{$job}->{ready};
6881                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6882                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6883                     my $remaining = $total - $transferred;
6884                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6885
6886                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6887                 }
6888
6889                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6890             }
6891
6892             last if scalar(keys %$jobs) == 0;
6893
6894             if ($readycounter == scalar(keys %$jobs)) {
6895                 print "all mirroring jobs are ready \n";
6896                 last if $skipcomplete; #do the complete later
6897
6898                 if ($vmiddst && $vmiddst != $vmid) {
6899                     my $agent_running = $qga && qga_check_running($vmid);
6900                     if ($agent_running) {
6901                         print "freeze filesystem\n";
6902                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6903                     } else {
6904                         print "suspend vm\n";
6905                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6906                     }
6907
6908                     # if we clone a disk for a new target vm, we don't switch the disk
6909                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6910
6911                     if ($agent_running) {
6912                         print "unfreeze filesystem\n";
6913                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6914                     } else {
6915                         print "resume vm\n";
6916                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6917                     }
6918
6919                     last;
6920                 } else {
6921
6922                     foreach my $job (keys %$jobs) {
6923                         # try to switch the disk if source and destination are on the same guest
6924                         print "$job: Completing block job...\n";
6925
6926                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6927                         if ($@ =~ m/cannot be completed/) {
6928                             print "$job: Block job cannot be completed, try again.\n";
6929                             $err_complete++;
6930                         }else {
6931                             print "$job: Completed successfully.\n";
6932                             $jobs->{$job}->{complete} = 1;
6933                         }
6934                     }
6935                 }
6936             }
6937             sleep 1;
6938         }
6939     };
6940     my $err = $@;
6941
6942     if ($err) {
6943         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6944         die "mirroring error: $err";
6945     }
6946
6947 }
6948
6949 sub qemu_blockjobs_cancel {
6950     my ($vmid, $jobs) = @_;
6951
6952     foreach my $job (keys %$jobs) {
6953         print "$job: Cancelling block job\n";
6954         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6955         $jobs->{$job}->{cancel} = 1;
6956     }
6957
6958     while (1) {
6959         my $stats = mon_cmd($vmid, "query-block-jobs");
6960
6961         my $running_jobs = {};
6962         foreach my $stat (@$stats) {
6963             $running_jobs->{$stat->{device}} = $stat;
6964         }
6965
6966         foreach my $job (keys %$jobs) {
6967
6968             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6969                 print "$job: Done.\n";
6970                 delete $jobs->{$job};
6971             }
6972         }
6973
6974         last if scalar(keys %$jobs) == 0;
6975
6976         sleep 1;
6977     }
6978 }
6979
6980 sub clone_disk {
6981     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6982         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6983
6984     my $newvolid;
6985
6986     if (!$full) {
6987         print "create linked clone of drive $drivename ($drive->{file})\n";
6988         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6989         push @$newvollist, $newvolid;
6990     } else {
6991
6992         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6993         $storeid = $storage if $storage;
6994
6995         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6996         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6997
6998         print "create full clone of drive $drivename ($drive->{file})\n";
6999         my $name = undef;
7000         if (drive_is_cloudinit($drive)) {
7001             $name = "vm-$newvmid-cloudinit";
7002             $name .= ".$dst_format" if $dst_format ne 'raw';
7003             $snapname = undef;
7004             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7005         }
7006         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7007         push @$newvollist, $newvolid;
7008
7009         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7010
7011         if (drive_is_cloudinit($drive)) {
7012             goto no_data_clone;
7013         }
7014
7015         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7016         if (!$running || $snapname) {
7017             # TODO: handle bwlimits
7018             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7019         } else {
7020
7021             my $kvmver = get_running_qemu_version ($vmid);
7022             if (!min_version($kvmver, 2, 7)) {
7023                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7024                     if $drive->{iothread};
7025             }
7026
7027             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7028         }
7029     }
7030
7031 no_data_clone:
7032     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7033
7034     my $disk = $drive;
7035     $disk->{format} = undef;
7036     $disk->{file} = $newvolid;
7037     $disk->{size} = $size;
7038
7039     return $disk;
7040 }
7041
7042 sub get_running_qemu_version {
7043     my ($vmid) = @_;
7044     my $res = mon_cmd($vmid, "query-version");
7045     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7046 }
7047
7048 sub qemu_use_old_bios_files {
7049     my ($machine_type) = @_;
7050
7051     return if !$machine_type;
7052
7053     my $use_old_bios_files = undef;
7054
7055     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7056         $machine_type = $1;
7057         $use_old_bios_files = 1;
7058     } else {
7059         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7060         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7061         # load new efi bios files on migration. So this hack is required to allow
7062         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7063         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7064         $use_old_bios_files = !min_version($version, 2, 4);
7065     }
7066
7067     return ($use_old_bios_files, $machine_type);
7068 }
7069
7070 sub create_efidisk($$$$$) {
7071     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7072
7073     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7074     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7075
7076     my $vars_size_b = -s $ovmf_vars;
7077     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7078     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7079     PVE::Storage::activate_volumes($storecfg, [$volid]);
7080
7081     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7082     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7083
7084     return ($volid, $size/1024);
7085 }
7086
7087 sub vm_iothreads_list {
7088     my ($vmid) = @_;
7089
7090     my $res = mon_cmd($vmid, 'query-iothreads');
7091
7092     my $iothreads = {};
7093     foreach my $iothread (@$res) {
7094         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7095     }
7096
7097     return $iothreads;
7098 }
7099
7100 sub scsihw_infos {
7101     my ($conf, $drive) = @_;
7102
7103     my $maxdev = 0;
7104
7105     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7106         $maxdev = 7;
7107     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7108         $maxdev = 1;
7109     } else {
7110         $maxdev = 256;
7111     }
7112
7113     my $controller = int($drive->{index} / $maxdev);
7114     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7115
7116     return ($maxdev, $controller, $controller_prefix);
7117 }
7118
7119 sub add_hyperv_enlightenments {
7120     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7121
7122     return if $winversion < 6;
7123     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7124
7125     if ($gpu_passthrough || defined($hv_vendor_id)) {
7126         $hv_vendor_id //= 'proxmox';
7127         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7128     }
7129
7130     if (min_version($machine_version, 2, 3)) {
7131         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7132         push @$cpuFlags , 'hv_vapic';
7133         push @$cpuFlags , 'hv_time';
7134     } else {
7135         push @$cpuFlags , 'hv_spinlocks=0xffff';
7136     }
7137
7138     if (min_version($machine_version, 2, 6)) {
7139         push @$cpuFlags , 'hv_reset';
7140         push @$cpuFlags , 'hv_vpindex';
7141         push @$cpuFlags , 'hv_runtime';
7142     }
7143
7144     if ($winversion >= 7) {
7145         push @$cpuFlags , 'hv_relaxed';
7146
7147         if (min_version($machine_version, 2, 12)) {
7148             push @$cpuFlags , 'hv_synic';
7149             push @$cpuFlags , 'hv_stimer';
7150         }
7151
7152         if (min_version($machine_version, 3, 1)) {
7153             push @$cpuFlags , 'hv_ipi';
7154         }
7155     }
7156 }
7157
7158 sub windows_version {
7159     my ($ostype) = @_;
7160
7161     return 0 if !$ostype;
7162
7163     my $winversion = 0;
7164
7165     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7166         $winversion = 5;
7167     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7168         $winversion = 6;
7169     } elsif ($ostype =~ m/^win(\d+)$/) {
7170         $winversion = $1;
7171     }
7172
7173     return $winversion;
7174 }
7175
7176 sub resolve_dst_disk_format {
7177         my ($storecfg, $storeid, $src_volname, $format) = @_;
7178         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7179
7180         if (!$format) {
7181             # if no target format is specified, use the source disk format as hint
7182             if ($src_volname) {
7183                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7184                 $format = qemu_img_format($scfg, $src_volname);
7185             } else {
7186                 return $defFormat;
7187             }
7188         }
7189
7190         # test if requested format is supported - else use default
7191         my $supported = grep { $_ eq $format } @$validFormats;
7192         $format = $defFormat if !$supported;
7193         return $format;
7194 }
7195
7196 sub resolve_first_disk {
7197     my $conf = shift;
7198     my @disks = PVE::QemuServer::valid_drive_names();
7199     my $firstdisk;
7200     foreach my $ds (reverse @disks) {
7201         next if !$conf->{$ds};
7202         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7203         next if PVE::QemuServer::drive_is_cdrom($disk);
7204         $firstdisk = $ds;
7205     }
7206     return $firstdisk;
7207 }
7208
7209 sub generate_uuid {
7210     my ($uuid, $uuid_str);
7211     UUID::generate($uuid);
7212     UUID::unparse($uuid, $uuid_str);
7213     return $uuid_str;
7214 }
7215
7216 sub generate_smbios1_uuid {
7217     return "uuid=".generate_uuid();
7218 }
7219
7220 sub nbd_stop {
7221     my ($vmid) = @_;
7222
7223     mon_cmd($vmid, 'nbd-server-stop');
7224 }
7225
7226 sub create_reboot_request {
7227     my ($vmid) = @_;
7228     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7229         or die "failed to create reboot trigger file: $!\n";
7230     close($fh);
7231 }
7232
7233 sub clear_reboot_request {
7234     my ($vmid) = @_;
7235     my $path = "/run/qemu-server/$vmid.reboot";
7236     my $res = 0;
7237
7238     $res = unlink($path);
7239     die "could not remove reboot request for $vmid: $!"
7240         if !$res && $! != POSIX::ENOENT;
7241
7242     return $res;
7243 }
7244
7245 # bash completion helper
7246
7247 sub complete_backup_archives {
7248     my ($cmdname, $pname, $cvalue) = @_;
7249
7250     my $cfg = PVE::Storage::config();
7251
7252     my $storeid;
7253
7254     if ($cvalue =~ m/^([^:]+):/) {
7255         $storeid = $1;
7256     }
7257
7258     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7259
7260     my $res = [];
7261     foreach my $id (keys %$data) {
7262         foreach my $item (@{$data->{$id}}) {
7263             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7264             push @$res, $item->{volid} if defined($item->{volid});
7265         }
7266     }
7267
7268     return $res;
7269 }
7270
7271 my $complete_vmid_full = sub {
7272     my ($running) = @_;
7273
7274     my $idlist = vmstatus();
7275
7276     my $res = [];
7277
7278     foreach my $id (keys %$idlist) {
7279         my $d = $idlist->{$id};
7280         if (defined($running)) {
7281             next if $d->{template};
7282             next if $running && $d->{status} ne 'running';
7283             next if !$running && $d->{status} eq 'running';
7284         }
7285         push @$res, $id;
7286
7287     }
7288     return $res;
7289 };
7290
7291 sub complete_vmid {
7292     return &$complete_vmid_full();
7293 }
7294
7295 sub complete_vmid_stopped {
7296     return &$complete_vmid_full(0);
7297 }
7298
7299 sub complete_vmid_running {
7300     return &$complete_vmid_full(1);
7301 }
7302
7303 sub complete_storage {
7304
7305     my $cfg = PVE::Storage::config();
7306     my $ids = $cfg->{ids};
7307
7308     my $res = [];
7309     foreach my $sid (keys %$ids) {
7310         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7311         next if !$ids->{$sid}->{content}->{images};
7312         push @$res, $sid;
7313     }
7314
7315     return $res;
7316 }
7317
7318 sub complete_migration_storage {
7319     my ($cmd, $param, $current_value, $all_args) = @_;
7320
7321     my $targetnode = @$all_args[1];
7322
7323     my $cfg = PVE::Storage::config();
7324     my $ids = $cfg->{ids};
7325
7326     my $res = [];
7327     foreach my $sid (keys %$ids) {
7328         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7329         next if !$ids->{$sid}->{content}->{images};
7330         push @$res, $sid;
7331     }
7332
7333     return $res;
7334 }
7335
7336 1;